def _load_child_cfg(self, entry, mk_config):
'''Load a child configuration for a primary cfg'''
if type(entry) is str:
# Treat this as a file entry. Substitute wildcards in cfg_file
# files since we need to process them right away.
cfg_file = subst_wildcards(entry, self.__dict__, ignore_error=True)
self.cfgs.append(self.create_instance(mk_config, cfg_file))
elif type(entry) is dict:
# Treat this as a cfg expanded in-line
temp_cfg_file = self._conv_inline_cfg_to_hjson(entry)
if not temp_cfg_file:
return
self.cfgs.append(self.create_instance(mk_config, temp_cfg_file))
# Delete the temp_cfg_file once the instance is created
try:
log.log(VERBOSE, "Deleting temp cfg file:\n%s", temp_cfg_file)
os.system("/bin/rm -rf " + temp_cfg_file)
except IOError:
log.error("Failed to remove temp cfg file:\n%s", temp_cfg_file)
else:
log.error(
"Type of entry \"%s\" in the \"use_cfgs\" key is invalid: %s",
entry, str(type(entry)))
sys.exit(1)
def __init__(self, run_items, sim_cfg):
# Construct the cov_db_dirs right away from the run_items. This is a
# special variable used in the HJson.
self.cov_db_dirs = []
for run in run_items:
if run.cov_db_dir not in self.cov_db_dirs:
self.cov_db_dirs.append(run.cov_db_dir)
# Early lookup the cov_merge_db_dir, which is a mandatory misc
# attribute anyway. We need it to compute additional cov db dirs.
self.cov_merge_db_dir = subst_wildcards("{cov_merge_db_dir}",
sim_cfg.__dict__)
# Prune previous merged cov directories, keeping past 7 dbs.
prev_cov_db_dirs = clean_odirs(odir=self.cov_merge_db_dir, max_odirs=7)
# If the --cov-merge-previous command line switch is passed, then
# merge coverage with the previous runs.
if sim_cfg.cov_merge_previous:
self.cov_db_dirs += [str(item) for item in prev_cov_db_dirs]
super().__init__(sim_cfg)
self.dependencies += run_items
# Run coverage merge even if one test passes.
self.needs_all_dependencies_passing = False
# Append cov_db_dirs to the list of exports.
self.exports["cov_db_dirs"] = "\"{}\"".format(" ".join(
self.cov_db_dirs))
def _load_single_file(target, path, is_first, arg_keys):
'''Load a single hjson file, merging its keys into target
Returns a list of further includes that should be loaded.
'''
hjson = parse_hjson(path)
if not isinstance(hjson, dict):
raise RuntimeError(
'{!r}: Top-level hjson object is not a dictionary.'.format(path))
import_cfgs = []
for key, dict_val in hjson.items():
# If this key got set at the start of time and we want to ignore any
# updates: ignore them!
if key in arg_keys:
continue
# If key is 'import_cfgs', this should be a list. Add each item to the
# list of cfgs to process
if key == 'import_cfgs':
if not isinstance(dict_val, list):
raise RuntimeError('{!r}: import_cfgs value is {!r}, but '
'should be a list.'.format(path, dict_val))
import_cfgs += dict_val
continue
# 'use_cfgs' is a bit like 'import_cfgs', but is only used for primary
# config files (where it is a list of the child configs). This
# shouldn't be used except at top-level (the first configuration file
# to be loaded).
#
# If defined, check that it's a list, but then allow it to be set in
# the target dictionary as usual.
if key == 'use_cfgs':
if not is_first:
raise RuntimeError('{!r}: File is included by another one, '
'but defines "use_cfgs".'.format(path))
if not isinstance(dict_val, list):
raise RuntimeError(
'{!r}: use_cfgs must be a list. Saw {!r}.'.format(
path, dict_val))
# Otherwise, update target with this attribute
set_target_attribute(path, target, key, dict_val)
# Expand the names of imported configuration files as we return them
return [
subst_wildcards(cfg_path,
target,
ignored_wildcards=[],
ignore_error=False) for cfg_path in import_cfgs
]
def _gen_results(self):
# '''
# The function is called after the regression has completed. It looks
# for a regr_results.hjson file with aggregated results from the
# synthesis run. The hjson needs to have the following (potentially
# empty) fields
#
# results = {
# "tool": "dc",
# "top" : <name of toplevel>,
#
# "messages": {
# "flow_errors" : [],
# "flow_warnings" : [],
# "analyze_errors" : [],
# "analyze_warnings" : [],
# "elab_errors" : [],
# "elab_warnings" : [],
# "compile_errors" : [],
# "compile_warnings" : [],
# },
#
# "timing": {
# # per timing group (ususally a clock domain)
# # in nano seconds
# <group> : {
# "tns" : <value>,
# "wns" : <value>,
# "period" : <value>,
# ...
# }
# },
#
# "area": {
# # gate equivalent of a NAND2 gate
# "ge" : <value>,
#
# # summary report, in GE
# "comb" : <value>,
# "buf" : <value>,
# "reg" : <value>,
# "macro" : <value>,
# "total" : <value>,
#
# # hierchical report of first submodule level
# "instances" : {
# <name> : {
# "comb" : <value>,
# "buf" : <value>,
# "reg" : <value>,
# "macro" : <value>,
# "total" : <value>,
# },
# ...
# },
# },
#
# "power": {
# "net" : <value>,
# "int" : <value>,
# "leak" : <value>,
# },
#
# "units": {
# "voltage" : <value>,
# "capacitance" : <value>,
# "time" : <value>,
# "dynamic" : <value>,
# "static" : <value>,
# }
# }
#
# note that if this is a primary config, the results will
# be generated using the _gen_results_summary function
# '''
def _create_entry(val, norm=1.0, total=None, perctag="%"):
"""
Create normalized entry with an optional
percentage appended in brackets.
"""
if val is not None and norm is not None:
if total is not None:
perc = float(val) / float(total) * 100.0
entry = "%2.1f %s" % (perc, perctag)
else:
value = float(val) / norm
entry = "%2.1f" % (value)
else:
entry = "--"
return entry
self.result = {}
# Generate results table for runs.
results_str = "## " + self.results_title + "\n\n"
results_str += "### " + self.timestamp_long + "\n"
if self.revision:
results_str += "### " + self.revision + "\n"
results_str += "### Branch: " + self.branch + "\n"
results_str += "### Synthesis Tool: " + self.tool.upper() + "\n\n"
# TODO: extend this to support multiple build modes
for mode in self.build_modes:
# results_str += "## Build Mode: " + mode.name + "\n\n"
result_data = Path(
subst_wildcards(self.build_dir, {"build_mode": mode.name}) +
'/results.hjson')
log.info("looking for result data file at %s", result_data)
try:
with result_data.open() as results_file:
self.result = hjson.load(results_file, use_decimal=True)
except IOError as err:
log.warning("%s", err)
self.result = {
"messages": {
"flow_errors": ["IOError: %s" % err],
"flow_warnings": [],
"analyze_errors": [],
"analyze_warnings": [],
"elab_errors": [],
"elab_warnings": [],
"compile_errors": [],
"compile_warnings": [],
},
}
# Message summary
# results_str += "### Tool Message Summary\n\n"
if "messages" in self.result:
header = [
"Build Mode", "Flow Warnings", "Flow Errors",
"Analyze Warnings", "Analyze Errors", "Elab Warnings",
"Elab Errors", "Compile Warnings", "Compile Errors"
]
colalign = ("left", ) + ("center", ) * (len(header) - 1)
table = [header]
messages = self.result["messages"]
table.append([
mode.name,
str(len(messages["flow_warnings"])) + " W ",
str(len(messages["flow_errors"])) + " E ",
str(len(messages["analyze_warnings"])) + " W ",
str(len(messages["analyze_errors"])) + " E ",
str(len(messages["elab_warnings"])) + " W ",
str(len(messages["elab_errors"])) + " E ",
str(len(messages["compile_warnings"])) + " W ",
str(len(messages["compile_errors"])) + " E ",
])
if len(table) > 1:
results_str += tabulate(table,
headers="firstrow",
tablefmt="pipe",
colalign=colalign) + "\n\n"
else:
results_str += "No messages found\n\n"
else:
results_str += "No messages found\n\n"
# Hierarchical Area report
results_str += "### Circuit Complexity in [kGE]\n\n"
if "area" in self.result:
header = [
"Instance", "Comb ", "Buf/Inv", "Regs", "Macros", "Total",
"Total [%]"
]
colalign = ("left", ) + ("center", ) * (len(header) - 1)
table = [header]
# print top-level summary first
row = ["**" + self.result["top"] + "**"]
try:
kge = float(self.result["area"]["ge"]) * 1000.0
for field in ["comb", "buf", "reg", "macro", "total"]:
row += [
"**" +
_create_entry(self.result["area"][field], kge) +
"**"
]
row += ["**--**"]
table.append(row)
# go through submodules
for name in self.result["area"]["instances"].keys():
if name == self.result["top"]:
continue
row = [name]
for field in ["comb", "buf", "reg", "macro", "total"]:
row += [
_create_entry(
self.result["area"]["instances"][name]
[field], kge)
]
# add percentage of total
row += [
_create_entry(
self.result["area"]["instances"][name][field],
kge, self.result["area"]["total"], "%u")
]
table.append(row)
except TypeError:
results_str += "Gate equivalent is not properly defined\n\n"
if len(table) > 1:
results_str += tabulate(table,
headers="firstrow",
tablefmt="pipe",
colalign=colalign) + "\n\n"
else:
results_str += "No area report found\n\n"
else:
results_str += "No area report found\n\n"
# Timing report
results_str += "### Timing in [ns]\n\n"
if "timing" in self.result and "units" in self.result:
header = ["Clock", "Period", "WNS", "TNS"]
colalign = ("left", ) + ("center", ) * (len(header) - 1)
table = [header]
for clock in self.result["timing"].keys():
row = [clock]
row += [
_create_entry(
self.result["timing"][clock]["period"],
1.0E-09 / float(self.result["units"]["time"])),
_create_entry(
self.result["timing"][clock]["wns"], 1.0E-09 /
float(self.result["units"]["time"])) + " EN",
_create_entry(
self.result["timing"][clock]["tns"], 1.0E-09 /
float(self.result["units"]["time"])) + " EN"
]
table.append(row)
if len(table) > 1:
results_str += tabulate(table,
headers="firstrow",
tablefmt="pipe",
colalign=colalign) + "\n\n"
else:
results_str += "No timing report found\n\n"
else:
results_str += "No timing report found\n\n"
# Power report
results_str += "### Power Estimates in [mW]\n\n"
if "power" in self.result and "units" in self.result:
header = ["Network", "Internal", "Leakage", "Total"]
colalign = ("center", ) * len(header)
table = [header]
try:
self.result["power"]["net"]
power = [
float(self.result["power"]["net"]) *
float(self.result["units"]["dynamic"]),
float(self.result["power"]["int"]) *
float(self.result["units"]["dynamic"]),
float(self.result["power"]["leak"]) *
float(self.result["units"]["static"])
]
total_power = sum(power)
row = [
_create_entry(power[0], 1.0E-3) + " / " +
_create_entry(power[0], 1.0E-3, total_power),
_create_entry(power[1], 1.0E-3) + " / " +
_create_entry(power[1], 1.0E-3, total_power),
_create_entry(power[2], 1.0E-3) + " / " +
_create_entry(power[2], 1.0E-3, total_power),
_create_entry(total_power, 1.0E-3)
]
table.append(row)
# in case fp values are NoneType
except TypeError:
results_str += "No power report found\n\n"
if len(table) > 1:
results_str += tabulate(table,
headers="firstrow",
tablefmt="pipe",
colalign=colalign) + "\n\n"
else:
results_str += "No power report found\n\n"
# Append detailed messages if they exist
# Note that these messages are omitted in publication mode
hdr_key_pairs = [("Flow Warnings", "flow_warnings"),
("Flow Errors", "flow_errors"),
("Analyze Warnings", "analyze_warnings"),
("Analyze Errors", "analyze_errors"),
("Elab Warnings", "elab_warnings"),
("Elab Errors", "elab_errors"),
("Compile Warnings", "compile_warnings"),
("Compile Errors", "compile_errors")]
# Synthesis fails if any warning or error message has occurred
self.errors_seen = False
fail_msgs = ""
for _, key in hdr_key_pairs:
if key in self.result['messages']:
if self.result['messages'].get(key):
self.errors_seen = True
break
if self.errors_seen:
fail_msgs += "\n### Errors and Warnings for Build Mode `'" + mode.name + "'`\n"
for hdr, key in hdr_key_pairs:
msgs = self.result['messages'].get(key)
fail_msgs += print_msg_list("#### " + hdr, msgs,
self.max_msg_count)
# the email and published reports will default to self.results_md if they are
# empty. in case they need to be sanitized, override them and do not append
# detailed messages.
if self.sanitize_email_results:
self.email_results_md = results_str
if self.sanitize_publish_results:
self.publish_results_md = results_str
# locally generated result always contains all details
self.results_md = results_str + fail_msgs
# TODO: add support for pie / bar charts for area splits and
# QoR history
# Write results to the scratch area
results_file = self.scratch_path + "/results_" + self.timestamp + ".md"
with open(results_file, 'w') as f:
f.write(self.results_md)
log.log(VERBOSE, "[results page]: [%s] [%s]", self.name, results_file)
return self.results_md
def resolve_hjson_raw(self, hjson_dict):
attrs = self.__dict__.keys()
rm_hjson_dict_keys = []
import_cfgs = []
use_cfgs = []
for key in hjson_dict.keys():
if key in attrs:
hjson_dict_val = hjson_dict[key]
self_val = getattr(self, key)
scalar_types = {str: [""], int: [0, -1], bool: [False]}
# Case 1: key value in class and hjson_dict differ - error!
if type(hjson_dict_val) != type(self_val):
log.error("Conflicting key types: \"%s\" {\"%s, \"%s\"}",
key,
type(hjson_dict_val).__name__,
type(self_val).__name__)
sys.exit(1)
# Case 2: key value in class and hjson_dict are strs - set if
# not already set, else error!
elif type(hjson_dict_val) in scalar_types.keys():
defaults = scalar_types[type(hjson_dict_val)]
if self_val == hjson_dict_val:
rm_hjson_dict_keys.append(key)
elif self_val in defaults and hjson_dict_val not in defaults:
setattr(self, key, hjson_dict_val)
rm_hjson_dict_keys.append(key)
elif self_val not in defaults and hjson_dict_val not in defaults:
# check if key exists in command line args, use that, or
# throw conflicting error
# TODO, may throw the conflicting error but choose one and proceed rather
# than exit
override_with_args_val = False
if hasattr(self.args, key):
args_val = getattr(self.args, key)
if type(args_val) == str and args_val != "":
setattr(self, key, args_val)
override_with_args_val = True
if not override_with_args_val:
log.error(
"Conflicting values {\"%s\", \"%s\"} encountered for key \"%s\"",
str(self_val), str(hjson_dict_val), key)
sys.exit(1)
# Case 3: key value in class and hjson_dict are lists - merge'em
elif type(hjson_dict_val) is list and type(self_val) is list:
self_val.extend(hjson_dict_val)
setattr(self, key, self_val)
rm_hjson_dict_keys.append(key)
# Case 4: unknown issue
else:
log.error(
"Type of \"%s\" (%s) in %s appears to be invalid (should be %s)",
key,
type(hjson_dict_val).__name__, hjson_dict,
type(self_val).__name__)
sys.exit(1)
# If key is 'import_cfgs' then add to the list of cfgs to
# process
elif key == 'import_cfgs':
import_cfgs.extend(hjson_dict[key])
rm_hjson_dict_keys.append(key)
# If this is a master cfg list and the key is 'use_cfgs'
elif self.is_master_cfg and key == "use_cfgs":
use_cfgs.extend(hjson_dict[key])
# If this is a not master cfg list and the key is 'use_cfgs'
elif not self.is_master_cfg and key == "use_cfgs":
# Throw an error and exit
log.error(
"Key \"use_cfgs\" encountered in a non-master cfg file list \"%s\"",
self.flow_cfg_file)
sys.exit(1)
else:
# add key-value to class
setattr(self, key, hjson_dict[key])
rm_hjson_dict_keys.append(key)
# Parse imported cfgs
for cfg_file in import_cfgs:
if cfg_file not in self.imported_cfg_files:
self.imported_cfg_files.append(cfg_file)
# Substitute wildcards in cfg_file files since we need to process
# them right away.
cfg_file = subst_wildcards(cfg_file, self.__dict__)
self.parse_flow_cfg(cfg_file, False)
else:
log.error("Cfg file \"%s\" has already been parsed", cfg_file)
# Parse master cfg files
if self.is_master_cfg:
for entry in use_cfgs:
if type(entry) is str:
# Treat this as a file entry
# Substitute wildcards in cfg_file files since we need to process
# them right away.
cfg_file = subst_wildcards(entry,
self.__dict__,
ignore_error=True)
self.cfgs.append(self.create_instance(cfg_file))
elif type(entry) is dict:
# Treat this as a cfg expanded in-line
temp_cfg_file = self._conv_inline_cfg_to_hjson(entry)
if not temp_cfg_file:
continue
self.cfgs.append(self.create_instance(temp_cfg_file))
# Delete the temp_cfg_file once the instance is created
try:
log.log(VERBOSE, "Deleting temp cfg file:\n%s",
temp_cfg_file)
os.system("/bin/rm -rf " + temp_cfg_file)
except IOError:
log.error("Failed to remove temp cfg file:\n%s",
temp_cfg_file)
else:
log.error(
"Type of entry \"%s\" in the \"use_cfgs\" key is invalid: %s",
entry, str(type(entry)))
sys.exit(1)
def _gen_results(self, results):
# This function is called after the regression and looks for
# results.hjson file with aggregated results from the formal logfile.
# The hjson file is required to follow this format:
# {
# "messages": {
# "errors" : []
# "warnings" : []
# "cex" : ["property1", "property2"...],
# "undetermined": [],
# "unreachable" : [],
# },
#
# "summary": {
# "errors" : 0
# "warnings" : 2
# "proven" : 20,
# "cex" : 5,
# "covered" : 18,
# "undetermined": 7,
# "unreachable" : 2,
# "pass_rate" : "90 %",
# "cover_rate" : "90 %"
# },
# }
# The categories for property results are: proven, cex, undetermined,
# covered, and unreachable.
#
# If coverage was enabled then results.hjson will also have an item that
# shows formal coverage. It will have the following format:
# "coverage": {
# stimuli: "90 %",
# coi : "90 %",
# proof : "80 %"
# }
results_str = "## " + self.results_title + "\n\n"
results_str += "### " + self.timestamp_long + "\n"
if self.revision:
results_str += "### " + self.revision + "\n"
results_str += "### Branch: " + self.branch + "\n"
results_str += "### Tool: " + self.tool.upper() + "\n"
summary = [self.name] # cfg summary for publish results
assert len(self.deploy) == 1
mode = self.deploy[0]
if results[mode] == "P":
result_data = Path(
subst_wildcards(self.build_dir, {"build_mode": mode.name}),
'results.hjson')
try:
with open(result_data, "r") as results_file:
self.result = hjson.load(results_file, use_decimal=True)
except IOError as err:
log.warning("%s", err)
self.result = {
"messages": {
"errors": ["IOError: %s" % err],
}
}
results_str += "\n\n## Formal " + self.sub_flow.upper() + " Results\n"
formal_result_str, formal_summary = self.get_summary(self.result)
results_str += formal_result_str
summary += formal_summary
if self.cov:
results_str += "\n\n## Coverage Results\n"
results_str += ("### Coverage html file dir: " +
self.scratch_path + "/default/formal-icarus\n\n")
cov_result_str, cov_summary = self.get_coverage(self.result)
results_str += cov_result_str
summary += cov_summary
else:
summary += ["N/A", "N/A", "N/A"]
if results[mode] != "P":
results_str += "\n## List of Failures\n" + ''.join(
mode.launcher.fail_msg.message)
messages = self.result.get("messages")
if messages is not None:
results_str += self.parse_dict_to_str(messages)
self.results_md = results_str
# Generate result summary
self.result_summary[self.name] = summary
return self.results_md
def _gen_results(self):
# '''
# The function is called after the regression has completed. It looks
# for a regr_results.hjson file with aggregated results from the lint run.
# The hjson needs to have the following (potentially empty) fields
#
# {
# tool: ""
# errors: []
# warnings: []
# lint_errors: []
# lint_warning: []
# lint_infos: []
# }
#
# where each entry is a string representing a lint message. This allows
# to reuse the same LintCfg class with different tools since just the
# parsing script that transforms the tool output into the hjson above
# needs to be adapted.
#
# note that if this is a primary config, the results will
# be generated using the _gen_results_summary function
# '''
# Generate results table for runs.
results_str = "## " + self.results_title + "\n\n"
results_str += "### " + self.timestamp_long + "\n"
if self.revision:
results_str += "### " + self.revision + "\n"
results_str += "### Branch: " + self.branch + "\n"
results_str += "### Lint Tool: " + self.tool.upper() + "\n\n"
header = [
"Build Mode", "Tool Warnings", "Tool Errors", "Lint Warnings",
"Lint Errors"
]
colalign = ("center", ) * len(header)
table = [header]
# aggregated counts
self.result_summary["warnings"] = []
self.result_summary["errors"] = []
self.result_summary["lint_warnings"] = []
self.result_summary["lint_errors"] = []
fail_msgs = ""
for mode in self.build_modes:
result_data = Path(
subst_wildcards(self.build_dir, {"build_mode": mode.name}) +
'/results.hjson')
log.info("[results:hjson]: [%s]: [%s]", self.name, result_data)
try:
with result_data.open() as results_file:
self.result = hjson.load(results_file, use_decimal=True)
except IOError as err:
log.warning("%s", err)
self.result = {
"tool": "",
"errors": ["IOError: %s" % err],
"warnings": [],
"lint_errors": [],
"lint_warnings": [],
"lint_infos": []
}
if self.result:
table.append([
mode.name,
str(len(self.result["warnings"])) + " W ",
str(len(self.result["errors"])) + " E",
# We currently do not publish these infos at
# the moment len(self.result["lint_infos"]),
str(len(self.result["lint_warnings"])) + " W",
str(len(self.result["lint_errors"])) + " E"
])
else:
self.result = {
"tool": "",
"errors": [],
"warnings": [],
"lint_errors": [],
"lint_warnings": [],
"lint_infos": []
}
self.result_summary["warnings"] += self.result["warnings"]
self.result_summary["errors"] += self.result["errors"]
self.result_summary["lint_warnings"] += self.result[
"lint_warnings"]
self.result_summary["lint_errors"] += self.result["lint_errors"]
# Append detailed messages if they exist
hdr_key_pairs = [("Tool Warnings", "warnings"),
("Tool Errors", "errors"),
("Lint Warnings", "lint_warnings"),
("Lint Errors", "lint_errors")]
# Lint fails if any warning or error message has occurred
self.errors_seen = False
for _, key in hdr_key_pairs:
if key in self.result:
if self.result.get(key):
self.errors_seen = True
break
if self.errors_seen:
fail_msgs += "\n### Errors and Warnings for Build Mode `'" + mode.name + "'`\n"
for hdr, key in hdr_key_pairs:
msgs = self.result.get(key)
fail_msgs += print_msg_list("#### " + hdr, msgs,
self.max_msg_count)
if len(table) > 1:
self.results_md = results_str + tabulate(
table, headers="firstrow", tablefmt="pipe",
colalign=colalign) + "\n"
# the email and published reports will default to self.results_md if they are
# empty. in case they need to be sanitized, override them and do not append
# detailed messages.
if self.sanitize_email_results:
self.email_results_md = self.results_md
if self.sanitize_publish_results:
self.publish_results_md = self.results_md
# locally generated result always contains all details
self.results_md += fail_msgs
else:
self.results_md = results_str + "\nNo results to display.\n"
self.email_results_md = self.results_md
self.publish_results_md = self.results_md
# Write results to the scratch area
results_file = self.scratch_path + "/results_" + self.timestamp + ".md"
with open(results_file, 'w') as f:
f.write(self.results_md)
log.log(VERBOSE, "[results page]: [%s] [%s]", self.name, results_file)
return self.results_md
def _gen_results(self):
# This function is called after the regression and looks for
# results.hjson file with aggregated results from the FPV run.
# The hjson file is required to follow this format:
# {
# "messages": {
# "errors" : []
# "warnings" : []
# "cex" : ["property1", "property2"...],
# "undetermined": [],
# "unreachable" : [],
# },
#
# "fpv_summary": {
# "errors" : 0
# "warnings" : 2
# "proven" : 20,
# "cex" : 5,
# "covered" : 18,
# "undetermined": 7,
# "unreachable" : 2,
# "pass_rate" : "90 %",
# "cover_rate" : "90 %"
# },
# }
# The categories for property results are: proven, cex, undetermined,
# covered, and unreachable.
#
# If coverage was enabled then results.hjson will also have an item that
# shows FPV coverage. It will have the following format:
# "fpv_coverage": {
# stimuli: "90 %",
# coi : "90 %",
# proof : "80 %"
# }
results_str = "## " + self.results_title + "\n\n"
results_str += "### " + self.timestamp_long + "\n"
results_str += "### FPV Tool: " + self.tool.upper() + "\n"
results_str += "### LogFile dir: " + self.scratch_path + "/default\n\n"
summary = [self.name] # cfg summary for publish results
if len(self.build_modes) != 1:
mode_names = [mode.name for mode in self.build_modes]
log.error(
"FPV only supports mode 'default', but found these modes: %s",
mode_names)
else:
mode = self.build_modes[0]
result_data = Path(
subst_wildcards(self.build_dir, {"build_mode": mode.name}) +
'/results.hjson')
try:
with open(result_data, "r") as results_file:
self.result = hjson.load(results_file, use_decimal=True)
except IOError as err:
log.warning("%s", err)
self.result = {
"messages": {
"errors": ["IOError: %s" % err],
}
}
fpv_result_str, fpv_summary = self.get_fpv_summary_results(
self.result)
results_str += fpv_result_str
summary += fpv_summary
if self.cov:
results_str += "\n\n## Coverage Results\n"
results_str += ("### Coverage html file dir: " +
self.scratch_path +
"/default/formal-icarus\n\n")
cov_result_str, cov_summary = self.get_fpv_coverage_results(
self.result)
results_str += cov_result_str
summary += cov_summary
messages = self.result.get("messages")
if messages is not None:
results_str += self.parse_dict_to_str(messages)
# Write results to the scratch area
self.results_md = results_str
results_file = self.scratch_path + "/results_" + self.timestamp + ".md"
with open(results_file, 'w') as f:
f.write(self.results_md)
log.info("[results page]: [%s] [%s]", self.name, results_file)
# Generate result summary
if not self.cov:
summary += ["N/A", "N/A", "N/A"]
self.result_summary[self.name] = summary
return self.results_md
def _gen_results(self):
# '''
# The function is called after the regression has completed. It looks
# for a regr_results.hjson file with aggregated results from the lint run.
# The hjson needs to have the following (potentially empty) fields
#
# {
# tool: ""
# errors: []
# warnings: []
# lint_errors: []
# lint_warning: []
# lint_infos: []
# }
#
# where each entry is a string representing a lint message. This allows
# to reuse the same LintCfg class with different tools since just the
# parsing script that transforms the tool output into the hjson above
# needs to be adapted.
#
# note that if this is a master config, the results will
# be generated using the _gen_results_summary function
# '''
# Generate results table for runs.
results_str = "## " + self.results_title + "\n\n"
results_str += "### " + self.timestamp_long + "\n"
results_str += "### Lint Tool: " + self.tool.upper() + "\n\n"
header = [
"Build Mode", "Tool Warnings", "Tool Errors", "Lint Warnings",
"Lint Errors"
]
colalign = ("center", ) * len(header)
table = [header]
# aggregated counts
self.result_summary["warnings"] = []
self.result_summary["errors"] = []
self.result_summary["lint_warnings"] = []
self.result_summary["lint_errors"] = []
fail_msgs = ""
for mode in self.build_modes:
result_data = Path(
subst_wildcards(self.build_dir, {"build_mode": mode.name}) +
'/results.hjson')
log.info("looking for result data file at %s", result_data)
try:
with open(result_data, "r") as results_file:
self.result = hjson.load(results_file, use_decimal=True)
except IOError as err:
log.warning("%s", err)
self.result = {
"tool": "",
"errors": ["IOError: %s" % err],
"warnings": [],
"lint_errors": [],
"lint_warnings": [],
"lint_infos": []
}
if self.result:
table.append([
mode.name,
str(len(self.result["warnings"])) + " W ",
str(len(self.result["errors"])) + " E",
# We currently do not publish these infos at
# the moment len(self.result["lint_infos"]),
str(len(self.result["lint_warnings"])) + " W",
str(len(self.result["lint_errors"])) + " E"
])
else:
self.result = {
"tool": "",
"errors": [],
"warnings": [],
"lint_errors": [],
"lint_warnings": [],
"lint_infos": []
}
self.result_summary["warnings"] += self.result["warnings"]
self.result_summary["errors"] += self.result["errors"]
self.result_summary["lint_warnings"] += self.result[
"lint_warnings"]
self.result_summary["lint_errors"] += self.result["lint_errors"]
# Append detailed messages if they exist
if sum([
len(self.result["warnings"]),
len(self.result["errors"]),
len(self.result["lint_warnings"]),
len(self.result["lint_errors"])
]):
fail_msgs += "\n## Errors and Warnings for Build Mode `'" + mode.name + "'`\n"
fail_msgs += _print_msg_list("Tool Errors",
self.result["errors"])
fail_msgs += _print_msg_list("Tool Warnings",
self.result["warnings"])
fail_msgs += _print_msg_list("Lint Errors",
self.result["lint_errors"])
fail_msgs += _print_msg_list("Lint Warnings",
self.result["lint_warnings"])
# fail_msgs += _print_msg_list("Lint Infos", results["lint_infos"])
if len(table) > 1:
self.results_md = results_str + tabulate(
table, headers="firstrow", tablefmt="pipe",
colalign=colalign) + "\n" + fail_msgs
else:
self.results_md = results_str + "\nNo results to display.\n"
# Write results to the scratch area
self.results_file = self.scratch_path + "/results_" + self.timestamp + ".md"
with open(self.results_file, 'w') as f:
f.write(self.results_md)
log.info("[results page]: [%s] [%s]", self.name, results_file)
return self.results_md
def resolve_hjson_raw(self, path, hjson_dict):
import_cfgs = []
use_cfgs = []
for key, dict_val in hjson_dict.items():
# If key is 'import_cfgs' then add to the list of cfgs to process
if key == 'import_cfgs':
import_cfgs.extend(dict_val)
continue
# If the key is 'use_cfgs', we're only allowed to take effect for a
# primary config list. If we are in a primary config list, add it.
if key == 'use_cfgs':
if not self.is_primary_cfg:
log.error("Key 'use_cfgs' encountered in the non-primary "
"cfg file list {!r}.".format(path))
sys.exit(1)
use_cfgs.extend(dict_val)
continue
# Otherwise, set an attribute on self.
self._set_attribute(path, key, dict_val)
# Parse imported cfgs
for cfg_file in import_cfgs:
if cfg_file not in self.imported_cfg_files:
self.imported_cfg_files.append(cfg_file)
# Substitute wildcards in cfg_file files since we need to process
# them right away.
cfg_file = subst_wildcards(cfg_file, self.__dict__)
self._parse_cfg(cfg_file, False)
else:
log.error("Cfg file \"%s\" has already been parsed", cfg_file)
# Parse primary cfg files
if self.is_primary_cfg:
for entry in use_cfgs:
if type(entry) is str:
# Treat this as a file entry
# Substitute wildcards in cfg_file files since we need to process
# them right away.
cfg_file = subst_wildcards(entry,
self.__dict__,
ignore_error=True)
self.cfgs.append(self.create_instance(cfg_file))
elif type(entry) is dict:
# Treat this as a cfg expanded in-line
temp_cfg_file = self._conv_inline_cfg_to_hjson(entry)
if not temp_cfg_file:
continue
self.cfgs.append(self.create_instance(temp_cfg_file))
# Delete the temp_cfg_file once the instance is created
try:
log.log(VERBOSE, "Deleting temp cfg file:\n%s",
temp_cfg_file)
os.system("/bin/rm -rf " + temp_cfg_file)
except IOError:
log.error("Failed to remove temp cfg file:\n%s",
temp_cfg_file)
else:
log.error(
"Type of entry \"%s\" in the \"use_cfgs\" key is invalid: %s",
entry, str(type(entry)))
sys.exit(1)
def _gen_results(self, results):
# '''
# The function is called after the regression has completed. It looks
# for a results.hjson file with aggregated results from the lint run.
# The hjson needs to have the following format:
#
# {
# bucket_key: [str],
# // other buckets according to message_buckets configuration
# }
#
# Each bucket key points to a list of signatures (strings).
# The bucket categories and severities are defined in the
# message_buckets class variable, and can be set via Hjson Dvsim
# config files.
#
# Note that if this is a primary config, the results will
# be generated using the _gen_results_summary function
# '''
# Generate results table for runs.
results_str = f'## {self.results_title}\n\n'
results_str += f'### {self.timestamp_long}\n'
if self.revision:
results_str += f'### {self.revision}\n'
results_str += f'### Branch: {self.branch}\n'
results_str += f'### Tool: {self.tool.upper()}\n\n'
# Load all result files from all build modes and convert them to
# message buckets.
self.result = []
self.result_summary = MsgBuckets(self.message_buckets)
for mode in self.build_modes:
result_path = Path(
subst_wildcards(self.build_dir, {'build_mode': mode.name}) +
'/results.hjson')
log.info('[results:hjson]: [%s]: [%s]', self.name, result_path)
# TODO(#9079): replace this with native log parser
msgs = MsgBuckets(self.message_buckets)
msgs.load_hjson(result_path)
self.result.append(msgs)
# Aggregate with summary results
self.result_summary += msgs
# Construct Header
labels = self.result_summary.get_labels(self.report_severities)
header = ['Build Mode'] + labels
colalign = ('center', ) * len(header)
table = [header]
fail_msgs = ''
self.errors_seen = 0
keys = self.result_summary.get_keys(self.report_severities)
for mode, res in zip(self.build_modes, self.result):
row = [mode.name] + res.get_counts_md(keys)
table.append(row)
self.errors_seen += res.has_signatures(self.fail_severities)
fail_msgs += f"\n### Messages for Build Mode `'{mode.name}'`\n"
fail_msgs += res.print_signatures_md(self.report_severities,
self.max_msg_count)
if len(table) > 1:
self.results_md = results_str + tabulate(
table, headers='firstrow', tablefmt='pipe',
colalign=colalign) + '\n'
# The email and published reports will default to self.results_md
# if they are empty. In case they need to be sanitized, override
# them and do not append detailed messages.
if self.sanitize_email_results:
self.email_results_md = self.results_md
if self.sanitize_publish_results:
self.publish_results_md = self.results_md
# Locally generated result always contains all details.
self.results_md += fail_msgs
else:
self.results_md = f'{results_str}\nNo results to display.\n'
self.email_results_md = self.results_md
self.publish_results_md = self.results_md
return self.results_md
