def get_aid_grade(grade):
if grade[1:] in grades.aid:
gradeDict = {"string": grade, "value": grades.aid[grade[1:]]}
return gradeDict
else:
printer.error('parsing aid grade', grade)
return None
def get_components_to_build_list(builder_data_dict, parent_function):
# Call parent function
parent_function()
if builder_data_dict["components_to_build"] is not None:
# We remove the 'reboot' component because it will fail
# when umounting the 'BASE_DIRECTORY' because it is in use (the 'setenv.sh' file).
# The left the 'umount' component to do the work later within the 'collector' builder.
tools.remove_element(builder_data_dict["components_to_build"],
"reboot")
# Remove unnecessary components depending on the 'boot manager' selected
if config.EXCLUDED_BOOT_MANAGER.lower() == "sysv":
tools.remove_elements(builder_data_dict["components_to_build"],
builder_data_dict["sysv_components"])
elif config.EXCLUDED_BOOT_MANAGER.lower() == "systemd":
tools.remove_elements(builder_data_dict["components_to_build"],
builder_data_dict["systemd_components"])
else:
msg = "Unknown boot manager '{b}' selected"
msg = msg.format(b=config.EXCLUDED_BOOT_MANAGER)
printer.error()
def extract(filename, destination=None):
"""
Extract 'filename' compressed file to the 'destination' directory.
"""
if destination is None:
destination = os.getcwd()
# Show message info
msg = "Extracting file '{f}' into '{d}'"
msg = msg.format(f=os.path.basename(filename), d=destination)
printer.substep_info(msg)
# File extension
ext = os.path.splitext(filename)[1][1:]
if tarfile.is_tarfile(filename) is True:
extract_tarfile(filename, destination)
elif zipfile.is_zipfile(filename) is True:
extract_zipfile(filename, destination)
elif ext == "xz":
extract_tarfile(filename, destination, ext)
else:
msg = "tools.extract: archive '{f}' extension not recognized"
msg = msg.format(f=filename)
printer.error(msg)
def loop_over_feds(raw, book, adcTag="", **other):
okFeds = set()
adcs = set()
nTsMax = raw[None]["firstNTs"]
for fedId, dct in sorted(raw.iteritems()):
if fedId is None:
continue
book.fill(dct["nBytesSW"] / 1024.0, "nBytesSW_%d" % fedId, 64, 0, 32,
title="FED %d; kBytes;Events / bin" % fedId)
fedIdHw = dct["header"]["FEDid"]
if fedId != fedIdHw:
printer.error("FED %d has FEDid %d" % (fedId, fedIdHw))
continue
nBadHtrs, adcs1 = singleFedPlots(fedId=fedId, d=dct, book=book, nTsMax=nTsMax, **other)
adcs = adcs.union(adcs1)
if nBadHtrs:
return
okFeds.add(fedId)
h = raw[fedId]["header"]
if not h["utca"]:
checkHtrModules(fedId=fedId,
spigots=h["spigotList"],
htrBlocks=raw[fedId]["htrBlocks"])
if adcs:
book.fill(max(adcs), "max_adc_%s" % adcTag, 128, -0.5, 127.5,
title=";max ADC (when ErrF==0);Events / bin")
return okFeds
def read_recipe_file(component_name, directory="components"):
"""
Read YAML recipe file for the 'component_name' component. This recipe is placed
under the 'recipes' directory.
"""
filename = "{c}.yaml".format(c=component_name)
recipe_path = os.path.realpath(
os.path.join("recipes", directory, component_name, filename))
recipe_data = {}
if os.path.exists(recipe_path) is False:
# Try "yml" instead of "yaml"
filename = "{c}.yml".format(c=component_name)
recipe_path = os.path.realpath(
os.path.join("recipes", directory, component_name, filename))
# Error if do not exists
if (os.path.exists(recipe_path)) is False:
msg = "Error reading recipe. File '{f}' do not exists"
msg = msg.format(f=recipe_path)
printer.error(msg)
# File exists, so read it
fp = open(recipe_path, "r")
recipe_data = yaml.load(fp)
fp.close()
return recipe_data
def get_mixed_grade(grade):
if grade in grades.mixed:
gradeDict = {"string": grade, "value": grades.mixed[grade]}
return gradeDict
else:
printer.error('parsing mixed grade', grade)
return None
def get_snow_grade(grade):
if grade in grades.snow:
gradeDict = {"string": grade, "value": grades.snow[grade]}
return gradeDict
else:
printer.error('parsing snow grade', grade)
return None
def get_danger(danger):
if danger in grades.danger:
dangerDict = {"string": danger, "value": grades.danger[danger]}
return dangerDict
else:
printer.error('parsing danger', danger)
return None
def get_ice_grade(grade):
if grade in grades.ice:
gradeDict = {"string": grade, "value": grades.ice[grade]}
return gradeDict
else:
printer.error('parsing ice grade', grade)
return None
def fillEventMap(chain, iEntry, treeName, fedId0, branch0, s, kargs, forward,
forwardBcn, backward):
if treeName == "Events": # CMS CDAQ
rawThisFed = wordsOneFed(tree=chain,
fedId=fedId0,
collection=s["rawCollection"],
product=s["product"])
elif treeName == "CMSRAW": # HCAL local
rawThisFed = wordsOneChunk(tree=chain, branch=branch0)
else:
rawThisFed = wordsOneBranch(tree=chain, branch=branch0)
if rawThisFed is None:
sys.exit(" ")
raw = unpacked(fedData=rawThisFed, **kargs)
if not raw["nBytesSW"]:
printer.error(
"the first listed FED (%d) has zero bytes in tree '%s'." %
(fedId0, treeName))
return True # break!
evn, orn, bcn = coords(raw)
evnOrn = (evn, orn)
forward[iEntry] = evnOrn
forwardBcn[iEntry] = bcn
backward[evnOrn] = iEntry
def run_program_without_output(program_to_run, username=None):
"""
Run 'program_to_run' without printing anything to the standard output.
"""
if username is None:
p = run_program(program_to_run)
else:
p = run_program_as_user(program_to_run, username)
# Print to /dev/null
new_target = open(os.devnull, "w")
old_target = sys.stdout
sys.stdout = new_target
for line in iter(p.stdout.readline, b''):
# do nothing
continue
# Continue printing in console
sys.stdout = old_target
# Wait for the running command to finish
p.wait()
if p.returncode != 0:
msg = "Command '{pr}' failed to run. Return code: '{r}'"
msg = msg.format(pr=program_to_run, r=str(p.returncode))
printer.error(msg)
def uhtr(self):
if self.rbx[2] in "MP": # USC
end = "MP".index(self.rbx[2])
if self.sector == 18:
index = 0
else:
index = self.sector / 2
try:
# http://cmsdoc.cern.ch/cms/HCAL/document/CountingHouse/Crates/Crate_interfaces_2017.htm
crates = [30, 24, 20, 21, 25, 31, 35, 37, 34, 30] # 30 serves sectors 18 and 1
crate = crates[index]
slot1 = 6 * end + 3 * (self.sector % 2) + 2
except IndexError:
printer.error("Could not find uHTR reading out %s" % self.rbx)
return
else: # 904
ss = self.sector - 1
crate = 61 + ss / 9
if 9 <= ss:
ss -= 9
slot1 = 1 + 4 * ss / 3
link_status = uhtr_tool_link_status(crate, slot1)
for (crate, slot, ppod), lines in sorted(link_status.iteritems()):
print "Crate %d Slot %2d" % (crate, slot)
print lines
link, power, bad8b10b, bc0, _, write_delay, read_delay, fifo_occ, bprv, _, bad_full, invalid, _ = lines.split("\n")
# self.uhtr_compare(slot, ppod, power, 300.0, threshold=200.0)
self.uhtr_compare(slot, ppod, bad8b10b, 0)
self.uhtr_compare(slot, ppod, bc0, 1.12e1, threshold=0.1e1)
self.uhtr_compare(slot, ppod, fifo_occ, 11, threshold=8)
self.uhtr_compare(slot, ppod, bprv, 0x1111)
def continue_at(self):
"""
Start from the 'config.CONTINUE_AT' component of the first provided builder.
Fails in case this component do not exist.
"""
# .- is component present
if tools.is_element_present(
self.builder_data_dict["components_to_build"],
config.CONTINUE_AT) is True:
# get component index and trim 'components_to_build' list
index = tools.get_element_index(
self.builder_data_dict["components_to_build"],
config.CONTINUE_AT)
# trim list and update 'self.builder_data_dict' value
aux_list = self.builder_data_dict["components_to_build"][index:]
tools.add_to_dictionary(self.builder_data_dict,
"components_to_build",
aux_list,
concat=False)
# .- set 'config.CONTINUE_AT' to 'None' so we do not get into this method
# any more on the current execution
setattr(config, "CONTINUE_AT", None)
else:
text = """'continue-at' component '{c}' do not exists on the \
'components_to_build' list for the '{b}' builder""".format(
c=config.CONTINUE_AT, b=self.builder_data_dict["name"])
printer.error(text)
def loop_over_feds(raw, book, adcTag="", **other):
okFeds = set()
adcs = set()
for fedId, dct in sorted(raw.iteritems()):
if fedId is None:
continue
book.fill(dct["nBytesSW"] / 1024.0, "nBytesSW_%d" % fedId, 64, 0, 64,
title="FED %d; kBytes;Events / bin" % fedId)
fedIdHw = dct["header"]["FEDid"]
if fedId != fedIdHw:
printer.error("FED %d has FEDid %d" % (fedId, fedIdHw))
continue
nBadHtrs, adcs1 = singleFedPlots(fedId=fedId, d=dct, book=book, **other)
adcs = adcs.union(adcs1)
if nBadHtrs:
return
okFeds.add(fedId)
h = raw[fedId]["header"]
if not h["utca"]:
checkHtrModules(fedId=fedId,
spigots=h["spigotList"],
htrBlocks=raw[fedId]["htrBlocks"])
if adcs:
book.fill(max(adcs), "max_adc_%s" % adcTag, 128, -0.5, 127.5,
title=";max ADC (when ErrF==0);Events / bin")
return okFeds
def fillEventMap(chain, iEntry,
treeName, fedId0, branch0, s, kargs,
forward, forwardBcn, backward):
if treeName == "Events": # CMS CDAQ
rawThisFed = wordsOneFed(tree=chain,
fedId=fedId0,
collection=s["rawCollection"],
product=s["product"]
)
elif treeName == "CMSRAW": # HCAL local
rawThisFed = wordsOneChunk(tree=chain, branch=branch0)
else:
rawThisFed = wordsOneBranch(tree=chain, branch=branch0)
if rawThisFed is None:
sys.exit(" ")
raw = unpacked(fedData=rawThisFed, **kargs)
if not raw["nBytesSW"]:
printer.error("the first listed FED (%d) has zero bytes in tree '%s'." % (fedId0, treeName))
return True # break!
evn, orn, bcn = coords(raw)
evnOrn = (evn, orn)
forward[iEntry] = evnOrn
forwardBcn[iEntry] = bcn
backward[evnOrn] = iEntry
def opts():
parser = oparser(arg="RUN_NUMBER [RUN_NUMBER2 ...]")
look = optparse.OptionGroup(parser, "Options solely for use with look.py")
look.add_option("--quiet",
dest="quiet",
default=False,
action="store_true",
help="Print less to stdout")
look.add_option("--hhmm",
dest="hhmm",
default=None,
type="int",
help="minimum hhmm")
parser.add_option_group(look)
options, args = parser.parse_args()
runs = []
for arg in args:
try:
runs.append(int(arg))
except ValueError:
printer.warning("Could not convert '%s' to an integer." % arg)
if not runs:
printer.error("Please provide a run number as the argument.")
return options, runs
def command(p, cmd, timeout=5):
fields = cmd.split()
if not fields:
return None
if fields[0] == "jtag":
if len(fields) < 4:
print("COMMAND has to few fields: (%s)" % cmd)
return None
regexp = "(.*)%s %s %s# retcode=(.*)" % tuple(fields[1:])
else:
regexp = "{0}\s?#((\s|E)[^\r^\n]*)".format(re.escape(cmd))
try:
p.sendline(cmd)
p.expect(regexp, timeout=timeout)
return p.match.group(0).split("\r\n")
except pexpect.TIMEOUT:
tail = "tail -20 %s" % p.logfile.name
msg = printer.msg('The command "', p=False)
msg += printer.cyan(cmd, p=False)
msg += printer.msg(
'"\n produced unexpected output. Consult the log file, e.g.',
p=False)
msg += printer.msg('\n "%s" gives this:' %
printer.gray(tail, p=False),
p=False)
printer.error(msg)
os.system(tail)
sys.exit()
def get_rock_grade(grade):
grade = grade.lower()
if grade in grades.rock:
gradeDict = {"string": grade, "value": grades.rock[grade]}
return gradeDict
else:
printer.error('parsing rock grade', grade)
return None
def copy_file(src, dest):
"""
Copy 'src' file to 'dest'.
"""
if src != dest:
shutil.copyfile(src, dest)
else:
printer.error("tools.copy_file: 'src' and 'dest' are the same.")
def get_boulder_grade(grade):
nov = grade[1:].split("-")
if len(nov) > 1 and nov[1] == "easy":
value = grades.boulder[nov[1]]
else:
if nov[0] in grades.boulder:
value = grades.boulder[nov[0]]
else:
printer.error('parsing boulder grade', grade)
return None
gradeDict = {"string": grade, "value": value}
return gradeDict
def check_builders_dupes(self, bl=None):
"""
Check there is no a duplicate builder on 'self.builder_list'
"""
# 'bl' is not None if we are running this method from 'unittest'
if bl is not None:
self.builders_list = bl
for b in set(self.builders_list):
if self.builders_list.count(b) > 1:
msg = "Duplicated builder '{b}' is not allowed".format(b=b)
printer.error(msg)
def unpacked_sw_fed(fedData, header, nBytesPer, dump):
fedId = header["FEDid"]
if histo_fed(fedId):
return unpacked_histo(fedData, fedId, nBytesPer, dump)
else:
printer.error("Unpacking of software FED %d is not implemented." % fedId)
nBytesSW = fedData.size() * nBytesPer
return {"header": header,
"trailer": {"nWord64": nBytesSW // 8},
"other": {},
"nBytesSW": nBytesSW,
}
def read_file(filename, mode="r"):
"""
Read content from 'filename' if exists and return it content.
"""
# If file exists, read it
if os.path.exists(filename):
fp = open(filename, mode)
text = fp.read()
fp.close()
return text
else:
msg = "tools.read_file: file '{f}' not found".format(f=filename)
printer.error(msg)
def printChannelSummary(outputFile):
f = r.TFile(outputFile)
hs = []
for name in ["MatchedFibersCh%d" % i
for i in range(3)] + ["MatchedTriggerTowers"]:
h = f.Get(name)
if h:
hs.append(h)
lines = []
if hs:
words = ["nMatched"]
words += ["FibCh%d" % i for i in range(3)]
words.append(" TPs")
words = " ".join(words)
lines.append(words)
lines.append("-" * len(words))
else:
return
for iBin in range(0, 2 + hs[0].GetNbinsX()):
xs = [h.GetBinCenter(iBin) for h in hs]
if len(set(xs)) != 1:
printer.error("binnings for nMatched do not match.")
return
for h in hs:
w = h.GetBinWidth(iBin)
if 1.0e-6 < abs(w - 1.0):
printer.warning("Histogram %s bin %d has width %g" %
(h.GetName(), iBin, w))
x = xs[0]
cs = [h.GetBinContent(iBin) for h in hs]
if any(cs):
s = [" %4d" % x]
s += ["%4d" % c for c in cs]
lines.append(" ".join(s))
if len(lines) > 12:
printer.info(
"suppressed printing of match histogram (more than 10 different occupancies)"
)
else:
for line in lines:
print line
f.Close()
def build(self):
"""
'build' command.
"""
# 'build' command. It requires 'sudo' privileges to mount/umount directories
# and run admin commands such as 'chroot', 'loosetup' and others.
if os.getuid() != 0:
msg = "'build' command requires root privileges. Please try again using 'sudo'"
printer.error(msg)
# Parse command line arguments
self.build_parser = self.cli.configure_build_parser()
self.build_args = self.build_parser.parse_args(
self.all_args.command[1:])
# Set boolean configuration flags once arguments get actually parsed.
self.set_config_option(self.build_args)
# .- Check boot manager and meson builder combination
if tools.is_element_present(self.build_args.builders_list, "system") is True and \
tools.check_meson_builder_combination(m=config.INCLUDE_MESON_BUILDER,
sv=config.SYSV,
sd=config.SYSTEMD) is False:
printer.error(
"You can not use that combination of 'boot_manager' and 'meson builder'"
)
# Create 'config.BASE_DIRECTORY' if necessary
tools.create_directory(config.BASE_DIRECTORY)
# Create and build 'builders_list'
for builder in self.build_args.builders_list:
os.chdir(self.lfsbuilder_src_directory)
# Generate builder object from BuilderGenerator
bg = builders.BuilderGenerator(builder)
o = bg.get_builder_reference()
del bg
# Run the real builder
o.set_attributes()
o.build()
o.clean_workspace()
del o
# Set 'lfsbuilder_src_directory' permission back to the original 'uid' and 'gid'
tools.set_recursive_owner_and_group(self.lfsbuilder_src_directory,
self.actual_owner_uid,
self.actual_owner_gid)
def xMin_xMax(graph=None):
N = graph.GetN()
X = graph.GetX()
if not N:
printer.error("graph contains zero points.")
return 0.0, 0.0
x = [X[i] for i in range(N)]
xMin = min(x)
xMax = max(x)
delta = xMax - xMin
if delta:
tenPercent = 0.1 * delta
else:
tenPercent = 0.1 / 60.0 # 1/10 second
return (xMin - tenPercent, xMax + tenPercent)
def xMin_xMax(graph=None):
N = graph.GetN()
X = graph.GetX()
if not N:
printer.error("graph contains zero points.")
return 0.0, 0.0
x = [X[i] for i in range(N)]
xMin = min(x)
xMax = max(x)
delta = xMax - xMin
if delta:
tenPercent = 0.1 * delta
else:
tenPercent = 0.1 / 60.0 # 1/10 second
return (xMin - tenPercent, xMax + tenPercent)
def printChannelSummary(outputFile):
f = r.TFile(outputFile)
hs = []
for name in ["MatchedFibersCh%d" % i for i in range(3)] + ["MatchedTriggerTowers"]:
h = f.Get(name)
if h:
hs.append(h)
lines = []
if hs:
words = ["nMatched"]
words += ["FibCh%d" % i for i in range(3)]
words.append(" TPs")
words = " ".join(words)
lines.append(words)
lines.append("-" * len(words))
else:
return
for iBin in range(0, 2 + hs[0].GetNbinsX()):
xs = [h.GetBinCenter(iBin) for h in hs]
if len(set(xs)) != 1:
printer.error("binnings for nMatched do not match.")
return
for h in hs:
w = h.GetBinWidth(iBin)
if 1.0e-6 < abs(w - 1.0):
printer.warning("Histogram %s bin %d has width %g" % (h.GetName(), iBin, w))
x = xs[0]
cs = [h.GetBinContent(iBin) for h in hs]
if any(cs):
s = [" %4d" % x]
s += ["%4d" % c for c in cs]
lines.append(" ".join(s))
if len(lines) > 12:
printer.info("suppressed printing of match histogram (more than 10 different occupancies)")
else:
for line in lines:
print(line)
f.Close()
def checkHtrModules(fedId=None, spigots=[], htrBlocks={}):
crates = []
for iBlock, block in htrBlocks.iteritems():
if block["IsTTP"]:
continue
expected = patterns.expectedHtr(fedId, spigots[iBlock])
crates.append(block["Crate"])
bad = [block["Top"] != expected["Top"],
block["Slot"] != expected["Slot"],
]
if any(bad):
fields = (fedId, spigot, block["Crate"],
block["Slot"], block["Top"],
expected["Slot"], expected["Top"])
printer.error("FED %3d spigot %2d has moduleId decode to crate %2d slot %2d%s (expected slot %2d%s)" % fields)
if len(set(crates)) != 1:
printer.error("FED %s contains modules with crate labels %s." % (str(fedId), str(crates)))
def substitute_multiple_in_file(filename, substitution_list):
"""
Do multiple substitutions in file 'filename' from
'subsitution_list'.
'subsitution_list must have an even number or elements.
"""
# Check 'substitution_list' has an even number of elements.
if len(substitution_list) % 2 != 0:
msg = """The substitution list is not valid. Number of elements: \
{length}.
Please ensure you didn't miss any element, this parameter should have an \
even length."""
msg = msg.format(length=len(substitution_list))
printer.error(msg)
for old, new in zip(substitution_list[0::2], substitution_list[1::2]):
substitute_in_file(filename, old, new)
def test_one(self, pro):
proxy = {"http": "http://" + pro, "https": "https://" + pro}
m = 10
while m > 0:
try:
t = time.time()
r = requests.get(config["test"]["url"],
timeout=config["test"]["timeout"])
assert r.ok, "状态码错误"
info("[测试器] 代理可用" + pro.ljust(21) + " 请求用时" +
str(round(time.time() - t, 2)) + "s")
return
except Exception as e:
error("[测试器] 代理超时" + pro.ljust(22) + "准备重试")
errors += 1
continue
error("[测试器] 代理无法使用" + pro + "移除")
self.db.remove(pro)
def check_non_privileged_user():
# Create the config.NON_PRIVILEGED_USERNAME does not exist
if tools.check_user_exists(config.NON_PRIVILEGED_USERNAME) is False:
msg = "User '{user}' expecified in the 'config.NON_PRIVILEGED_USERNAME' \
variable doesn't exist"
msg = msg.format(user=config.NON_PRIVILEGED_USERNAME)
printer.error(msg)
# Create this user and group
printer.warning("User '{u}' does not exists. Creating it".format(
u=config.NON_PRIVILEGED_USERNAME))
cmd = "groupadd {U}".format(u=config.NON_PRIVILEGED_USERNAME)
tools.run_program_without_output(cmd)
cmd = "useradd -s /bin/bash -g {u} -m -k /dev/null {u}"
cmd = cmd.format(u=config.NON_PRIVILEGED_USERNAME)
tools.run_program_without_output(cmd)
printer.info("User '{u}' created".format(u=config.NON_PRIVILEGED_USERNAME))
def checkHtrModules(fedId=None, spigots=[], htrBlocks={}):
crates = []
for iBlock, block in htrBlocks.iteritems():
if block["IsTTP"]:
continue
spigot = spigots[iBlock]
expected = hw.expectedVmeHtr(fedId, spigot)
crates.append(block["Crate"])
bad = [block["Top"] != expected["Top"],
block["Slot"] != expected["Slot"],
]
if any(bad):
fields = (fedId, spigot, block["Crate"],
block["Slot"], block["Top"],
expected["Slot"], expected["Top"])
printer.error("FED %3d spigot %2d has moduleId decode to crate %2d slot %2d%s (expected slot %2d%s)" % fields)
if len(set(crates)) != 1:
printer.warning("FED %s contains modules with crate labels %s." % (str(fedId), str(crates)))
def run(self):
info("[测试器] 开始运行测试器")
if config["test"]["method"] == "no":
for x in self.db.getall():
self.test_one(x)
if self.errors >= self.thowsold and not self.parse:
if config["crawl"]["open"]:
error("[测试器] 爬取器已经运行，无法启动自动补足程序")
else:
t = threading.Thread(target=self.completion)
t.start()
elif config["test"]["method"] == "future":
with ThreadPoolExecutor(config["test"]["threads"]) as e:
e.map(self.test_one, self.db.getall())
if self.errors >= self.thowsold and not self.parse:
if config["crawl"]["open"]:
error("[测试器] 爬取器已经运行，无法启动自动补足程序")
else:
t = threading.Thread(target=self.completion)
t.start()
def get_commitment(routeType):
commitmentGrades = {
"I": 1,
"II": 2,
"III": 3,
"IV": 4,
"V": 5,
"VI": 6,
"VII": 7,
}
for part in routeType:
part = part.strip()
if "Grade" in part:
grade = part.split(' ')[1]
if grade in commitmentGrades:
gradeDict = {"string": grade, "value": commitmentGrades[grade]}
return gradeDict
else:
printer.error('parsing commitment', grade)
return None
return None
def download_source(self):
"""
Download source code files.
"""
msg = "Download source code for '{n}'"
msg = msg.format(n=self.downloader_data["name"])
printer.info(msg)
# Create 'sources' directory recursively so the method can be call
# directly from unittest file 'tests/test_downloader.py'
tools.create_directory_recursive(self.downloader_data["lfsbuilder_sources_directory"])
# Download for selected builder
if self.downloader_data["name"] == "lfs":
self.download_lfs_sources()
elif self.downloader_data["name"] == "blfs":
self.download_blfs_sources()
else:
msg = "Downloading sources for '{b}' is not currently available"
msg = msg.format(b=self.downloader_data["name"])
printer.error(msg)
def printChannelSummary(outputFile):
f = r.TFile(outputFile)
hs = []
for name in ["MatchedFibersCh%d" % i for i in range(3)] + ["MatchedTriggerTowers"]:
h = f.Get(name)
if h:
hs.append(h)
if hs:
words = ["nMatched"]
words += ["FibCh%d" % i for i in range(3)]
words.append(" TPs")
words = " ".join(words)
print words
print "-" * len(words)
else:
return
for iBin in range(0, 2 + hs[0].GetNbinsX()):
xs = [h.GetBinCenter(iBin) for h in hs]
if len(set(xs)) != 1:
printer.error("binnings for nMatched do not match.")
return
for h in hs:
w = h.GetBinWidth(iBin)
if 1.0e-6 < abs(w - 1.0):
printer.warning("Histogram %s bin %d has width %g" % (h.GetName(), iBin, w))
x = xs[0]
cs = [h.GetBinContent(iBin) for h in hs]
if any(cs):
s = [" %4d" % x]
s += ["%4d" % c for c in cs]
print " ".join(s)
f.Close()
def htrHeaderV0(l={}, w=None, i=None, utca=None):
if i == 0:
l["EvN"] = w & 0xff
if i == 1:
l["EvN"] += w << 8
if i == 2 and not utca:
l["HM"] = (w >> 13) & 0x1
l["EE"] = (w >> 2) & 0x1
if i == 3:
l["OrN5"] = (w >> 11) & 0x1f
moduleId = w & 0x7ff
if utca:
l["Crate"] = moduleId >> 4
l["Slot"] = moduleId & 0xf
l["Top"] = " "
l["ModuleId"] = -1 # removed below
else:
# https://gitlab.cern.ch/cmshcos/hcal/blob/integration/hcalHW/src/common/hcalHTR.cc
# int id=(m_crate<<6)+((m_slot&0x1F)<<1)+((true_for_top)?(1):(0));
# fpga->dev->write("HTRsubmodN",id);
l["Crate"] = moduleId >> 6
l["Slot"] = (moduleId >> 1) & 0x1f
l["Top"] = "t" if (moduleId & 0x1) else "b"
l["ModuleId"] = moduleId # used for TTP below
if i == 4:
l["BcN"] = w & 0xfff
l["OrN5"], l["BcN"] = ornBcn(l["OrN5"], l["BcN"], utca)
l["FormatVer"] = (w >> 12) & 0xf
l["PayloadFormat"] = l["FormatVer"] # compat
l["UnsupportedFormat"] = (not utca) and (l["FormatVer"] != 6)
if i == 5:
if utca:
#l["nWord16Payload"] = w & 0x1fff # !document
l["nPreSamples"] = (w >> 3) & 0x1f # !document
else:
l["nWord16Tp"] = (w >> 8) & 0xff
l["nPreSamples"] = (w >> 3) & 0x1f
if i == 6:
l["US"] = (w >> 15) & 0x1
l["CM"] = (w >> 14) & 0x1
if i == 7:
l["IsIO"] = False
l["IsTTP"] = (w >> 15) & 0x1
l["PipelineLength"] = w & 0xff
if l["IsTTP"]:
l["TTPAlgo"] = (w >> 8) & 0x7
# https://gitlab.cern.ch/cmshcos/hcal/blob/integration/hcalTrig/src/common/hcalTechTrigProcessor.cc
# m_id.value()=m_slot*100+m_crateId;
l["Slot"] = l["ModuleId"] // 100
l["Crate"] = l["ModuleId"] % 100
l["Top"] = " "
del l["UnsupportedFormat"]
else:
l["FWFlavor"] = (w >> 8) & 0x7f
if l["UnsupportedFormat"]:
c = "(crate %2d slot %2d%1s)" % (l["Crate"], l["Slot"], l["Top"])
printer.error("HTR %s FormatVer %d is not supported." % (c, l["FormatVer"]))
del l["ModuleId"]
def singleFedPlots(fedId=None, d={}, book={}, **other):
book.fill(d["nWord16Skipped"], "nWord16Skipped_%d" % fedId, 14, -0.5, 13.5,
title="FED %d;nWord16 skipped during unpacking;Events / bin" % fedId)
h = d["header"]
t = d["trailer"]
if "TTS" in t:
book.fill(t["TTS"], "TTS_%d" % fedId, 16, -0.5, 15.5,
title="FED %d;TTS state;Events / bin" % fedId)
if "BcN" in h:
book.fill(h["BcN"] / 100.0, "BcN_%d" % fedId, hw.nBx, 0, hw.nBx / 100.0,
title="FED %d;BcN / 100;Events / bin" % fedId)
msg = "FED %d" % fedId
fedEvn = h.get("EvN")
fedOrn = h.get("OrN")
fedBcn = h.get("BcN")
if fedEvn is None:
printer.error("FED %d lacks EvN. Keys: %s" % str(h.keys()))
msg = ""
else:
msg = utils.coords(fedId, fedEvn, fedOrn, fedBcn)
htrOverviewBits(h, book, fedId, msg=msg, warn=other["warn"])
fedTime = utils.minutes(fedOrn, fedBcn)
nBadHtrs, ErrF, caps, adcs, fracEvN, fracADC, fracTP = htrSummary(blocks=d["htrBlocks"].values(),
book=book,
fedId=fedId,
fedEvn=fedEvn,
fedOrn5=fedOrn & 0x1f,
fedBcn=fedBcn,
msg=msg,
fedTime=fedTime,
**other)
errFSum = 0.0 + sum(ErrF.values())
if errFSum:
for code, n in ErrF.iteritems():
title = "FED %d;frac. chan. with ErrF == %d;Events / bin" % (fedId, code)
book.fill(n/errFSum, "ErrF%d_%d" % (code, fedId), 44, 0.0, 1.1, title=title)
frac0 = ErrF[0] / errFSum
else:
frac0 = -999. # dummy
book.fillGraph((fedTime, frac0), "frac0_vs_time_%d" % fedId,
title=("FED %d" % fedId) +
";time (minutes);frac. chan. with ErrF == 0")
if fracEvN is not None:
book.fillGraph((fedTime, fracEvN), "fracEvN_vs_time_%d" % fedId,
title=("FED %d" % fedId) +
";time (minutes);frac. HTRs with EvN == FED's")
if fracADC is not None:
book.fillGraph((fedTime, fracADC), "ADC_misMatch_vs_time_%d" % fedId,
title=("FED %d" % fedId) +
";time (minutes);#splitline{(# ch. match. ADCs) /}{(# ch. match. + ch. mismatch.)}")
if fracTP is not None:
book.fillGraph((fedTime, fracTP), "TP_misMatch_vs_time_%d" % fedId,
title=("FED %d" % fedId) +
";time (minutes);#splitline{(# match. TPs) /}{(# match. + mismatch. TPs)}")
if "nWord64" in t:
book.fillGraph((fedTime, t["nWord64"] * 8.0 / 1024), "kB_vs_time_%d" % fedId,
title=("FED %d" % fedId) +
";time (minutes);kB")
book.fill((fedBcn / 100.0, frac0), "frac0_vs_BcN_%d" % fedId,
(hw.nBx, 6), (0.0, 0.0), (hw.nBx / 100.0, 1.2),
title=";FED %d (BcN / 100);frac. chan. with ErrF == 0;Events / bin" % fedId)
capSum = 0.0+sum(caps.values())
if capSum:
book.fill(max(caps.values())/capSum, "PopCapFrac_%d" % fedId,
44, 0.0, 1.1,
title=("FED %d" % fedId) +
";frac. ErrF=0 chans w/most pop. capId;Events / bin"
)
book.fill(nBadHtrs, "nBadHtrs_%d" % fedId, 16, -0.5, 15.5,
title="FED %d; N bad HTRs;Events / bin" % fedId)
return nBadHtrs, adcs
def plotGlobal(f, pad, offset=None, names=[], logY=False, logX=False, logZ=True,
gopts="colz", feds1=[], feds2=[], doYx=True, retitle=True,
gridX=False, gridY=False, boxes=False, denoms={}):
keep = []
if denoms:
logZ = False
for iHisto, name in enumerate(names):
if not name:
continue
pad.cd(offset + iHisto)
adjustPad(logX=logX, logY=logY, logZ=logZ,
gridX=gridX, gridY=gridY)
h = f.Get(name)
if not h:
continue
shiftFlows(h)
zTitle = "Samples / bin"
if denoms.get(name):
denom = f.Get(denoms[name])
if not denom:
printer.error("%s not found" % denoms[name])
continue
shiftFlows(denom)
h = divided(h, denom)
zTitle = "# (%s) / # (%s)" % (h.GetTitle(), denom.GetTitle())
h.GetZaxis().SetTitle(zTitle)
else:
denom = None
h.Draw(gopts)
stylize(h)
magnify(h, factor=1.8)
if denom:
iGray = 56
colors = [19] * (iGray - 51) + range(iGray, 51 + 50)
r.gStyle.SetPalette(len(colors), array.array('i', colors))
h.GetZaxis().SetRangeUser(-0.1, 1.1)
nContours = 12
r.gStyle.SetNumberContours(nContours) # restored in pageTwo()
h.SetContour(nContours)
else:
h.GetZaxis().SetRangeUser(0.5, h.GetMaximum())
if boxes:
keep += drawCrates()
if retitle:
P = name[:name.find("_vs_")].upper()
h.GetXaxis().SetTitle("%s" % fedString(feds1))
h.GetYaxis().SetTitle("%s" % fedString(feds2))
h.GetZaxis().SetTitle(zTitle)
title = P
if h.GetTitle():
note = h.GetTitle().replace("SOI; ", "")
if note:
h.GetXaxis().SetTitle("%s (%s)" % (h.GetXaxis().GetTitle(), note))
if h.GetTitle().startswith("SOI"):
title += " (SOI)"
h.SetTitle(title)
xMin = h.GetXaxis().GetXmin()
xMax = h.GetXaxis().GetXmax()
line = r.TLine()
line.SetLineWidth(1)
line.SetLineStyle(2)
# if doYx:
# x0 = line.DrawLine(0.0, xMin, 0.0, xMax)
# y0 = line.DrawLine(xMin, 0.0, xMax, 0.0)
# keep += [x0, y0]
h.Draw(gopts + "same") # draw again to be on top of dashed lines
yx = r.TF1("yx", "x", -0.5, xMax)
yx.SetNpx(int(0.5 + xMax))
yx.SetLineColor(r.kBlack)
yx.SetLineWidth(1)
yx.SetLineStyle(3)
if doYx:
yx.Draw("same")
leg = r.TLegend(0.25, 0.73, 0.4, 0.85)
leg.SetBorderSize(0)
leg.SetFillStyle(0)
if doYx:
leg.AddEntry(yx, "y = x", "l")
# leg.AddEntry(x0, "zero", "l")
leg.Draw()
keep += [h, yx, leg]
# if name == "adc_vs_adc":
# h.SetTitle("")
# h.GetXaxis().SetTitle("ADC (VME FEDs 718,719)")
# h.GetYaxis().SetTitle("ADC (uTCA FED 1118)")
# r.gPad.Print("adc.pdf")
# import os
# os.system("pdfcrop adc.pdf")
return keep
def pageOne(f=None, feds1=[], feds2=[], canvas=None, pdf="", title=""):
pad20 = r.TPad("pad20", "pad20", 0.00, 0.00, 1.00, 1.00)
pad20.Divide(5, 4, 0.001, 0.001)
pad20.Draw()
keep = []
sfx = suffix(feds1, feds2)
cats = f.Get("category_vs_time" + sfx)
if not cats:
return
if not cats.GetN():
return
counts = yCounts(cats)
if 2 <= len(counts.keys()):
relabel(cats, counts)
keep += draw_graph(cats, title=title)
else:
ratemax = 5.0e7
evn_graph = f.Get("evn_vs_time" + sfx)
retitle(evn_graph)
if "/239/895" in evn_graph.GetTitle():
graph4 = big_clean(size=f.Get("kB_vs_time_%d" % feds1[0]),
frac0=f.Get("frac0_vs_time_%d" % feds1[0]),
sizeMin=0.8,
frac0Min=0.2,
height=ratemax / 5.0)
else:
graph4 = None
keep += draw_graph(graph=evn_graph,
title=title, ratemax=ratemax,
graph2=f.Get("bcn_delta_vs_time" + sfx),
graph3=resyncs(f.Get("incr_evn_vs_time" + sfx), ratemax),
graph4=graph4,
)
# single FED
keep += plotList(f, pad20, offset=5,
names=["BcN",
"nBytesSW", "ChannelFlavor", "nQieSamples", "nTpSamples", "htrOverviewBits",
"EvN_HTRs", "OrN5_HTRs", "BcN_HTRs", "ErrF0",
# "TTS", "PopCapFrac",
], feds1=feds1, feds2=feds2)
# EvN, OrN, BcN agreement
if not feds1:
printer.error(" in graphs.py pageOne: feds1 = %s" % str(feds1))
return
fed1 = sorted(feds1)[0]
for i, fed2 in enumerate(feds2[:3]):
pad20.cd(16 + i)
adjustPad(logY=True)
keep += histoLoop(f,
[("EvN", r.kBlue, 1),
("OrN", r.kCyan, 2),
("BcN", r.kBlack, 3),
],
lambda x: "delta%s_%s_%s" % (x, fed1, fed2),
)
# TS
pad20.cd(15)
adjustPad(logY=True)
keep += histoLoop(f,
[("nTS_for_matching_ADC", r.kBlue, 1),
("nTS_for_matching_TP", r.kCyan, 2),
],
lambda x: x,
)
# fibers
pad20.cd(19)
adjustPad(logY=True)
keep += histoLoop(f,
[("MatchedFibersCh0", r.kBlue, 1),
("MatchedFibersCh1", r.kCyan, 2),
("MatchedFibersCh2", r.kBlack, 3),
("MatchedTriggerTowers", r.kGreen, 4),
],
lambda x: x,
)
pad20.cd(20)
adjustPad(logY=True)
keep += histoLoop(f,
[("MisMatchedFibersCh0", r.kBlue, 1),
("MisMatchedFibersCh1", r.kCyan, 2),
("MisMatchedFibersCh2", r.kBlack, 3),
("MisMatchedTriggerTowers", r.kGreen, 4),
],
lambda x: x,
)
canvas.Print(pdf)