def find_latest_run_dir(config):
""" Find the latest run directory for given configuration """
task_dir = find_task_dir(config)
run_dir = get_latest_run_dir(task_dir)
if not run_dir:
raise InspectError(
"Failed to find run directory for task '{}' in '{}'".format(config.task_name, task_dir)
)
assert Path(run_dir).is_dir()
return run_dir
def resolve_vtr_source_file(config, filename, base_dir=""):
"""
Resolves an filename with a base_dir
Checks the following in order:
1) filename as absolute path
2) filename under config directory
3) base_dir as absolute path (join filename with base_dir)
4) filename and base_dir are relative paths (join under vtr_root)
"""
# Absolute path
if PurePath(filename).is_absolute():
return filename
# Under config
config_path = Path(config.config_dir)
assert config_path.is_absolute()
joined_path = config_path / filename
if joined_path.exists():
return str(joined_path)
# Under base dir
base_path = Path(base_dir)
if base_path.is_absolute():
# Absolute base
joined_path = base_path / filename
if joined_path.exists():
return str(joined_path)
else:
# Relative base under the VTR flow directory
joined_path = paths.vtr_flow_path / base_dir / filename
if joined_path.exists():
return str(joined_path)
# Not found
raise InspectError("Failed to resolve VTR source file {}".format(filename))
def check_two_files(
config,
first_results_filepath,
second_results_filepath,
first_name="task",
second_name="golden",
):
""" Compare two files results """
first_results = load_parse_results(first_results_filepath)
second_results = load_parse_results(second_results_filepath)
# Verify that the architecture and circuit are specified
for param in ["architecture", "circuit", "script_params"]:
if param not in first_results.PRIMARY_KEYS:
raise InspectError(
"Required param '{}' missing from {} results: {}".format(
param, first_name, first_results_filepath
),
first_results_filepath,
)
if param not in second_results.PRIMARY_KEYS:
raise InspectError(
"Required param '{}' missing from {} results: {}".format(
param, second_name, second_results_filepath
),
second_results_filepath,
)
# Verify that all params and pass requirement metric are included in both the result files
# We do not worry about non-pass_requriements elements being different or missing
pass_req_filepath = str(paths.pass_requirements_path / config.pass_requirements_file)
pass_requirements = load_pass_requirements(pass_req_filepath)
for metric in pass_requirements.keys():
for ((arch, circuit, script_params), result,) in first_results.all_metrics().items():
if metric not in result:
raise InspectError(
"Required metric '{}' missing from {} results".format(metric, first_name),
first_results_filepath,
)
for ((arch, circuit, script_params), result,) in second_results.all_metrics().items():
if metric not in result:
raise InspectError(
"Required metric '{}' missing from {} results".format(metric, second_name),
second_results_filepath,
)
# Load the primary keys for result files
second_primary_keys = []
for (arch, circuit, script_params), _ in second_results.all_metrics().items():
second_primary_keys.append((arch, circuit, script_params))
first_primary_keys = []
for (arch, circuit, script_params), _ in first_results.all_metrics().items():
first_primary_keys.append((arch, circuit, script_params))
# Ensure that first result file has all the second result file cases
for arch, circuit, script_params in second_primary_keys:
if first_results.metrics(arch, circuit, script_params) is None:
raise InspectError(
"Required case {}/{} missing from {} results: {}".format(
arch, circuit, first_name, first_results_filepath
)
)
# Warn about any elements in first result file that are not found in second result file
for arch, circuit, script_params in first_primary_keys:
if second_results.metrics(arch, circuit, script_params) is None:
print(
"Warning: {} includes result for {}/{} missing in {} results".format(
first_name, arch, circuit, second_name
)
)
num_qor_failures = 0
# Verify that the first results pass each metric for all cases in the second results
for (arch, circuit, script_params) in second_primary_keys:
second_metrics = second_results.metrics(arch, circuit, script_params)
first_metrics = first_results.metrics(arch, circuit, script_params)
first_fail = True
for metric in pass_requirements.keys():
if not metric in second_metrics:
print("Warning: Metric {} missing from {} results".format(metric, second_name))
continue
if not metric in first_metrics:
print("Warning: Metric {} missing from {} results".format(metric, first_name))
continue
try:
metric_passed, reason = pass_requirements[metric].check_passed(
second_metrics[metric], first_metrics[metric], second_name
)
except InspectError as error:
metric_passed = False
reason = error.msg
if not metric_passed:
if first_fail:
print(
"\n{}...[Fail]".format(
"/".join(str((Path(config.config_dir).parent)).split("/")[-3:])
)
)
first_fail = False
print("[Fail]\n{}/{}/{} {} {}".format(arch, circuit, script_params, metric, reason))
num_qor_failures += 1
return num_qor_failures