def find_latest_run_dir(config): """ Find the latest run directory for given configuration """ task_dir = find_task_dir(config) run_dir = get_latest_run_dir(task_dir) if not run_dir: raise InspectError( "Failed to find run directory for task '{}' in '{}'".format(config.task_name, task_dir) ) assert Path(run_dir).is_dir() return run_dir
def resolve_vtr_source_file(config, filename, base_dir=""): """ Resolves an filename with a base_dir Checks the following in order: 1) filename as absolute path 2) filename under config directory 3) base_dir as absolute path (join filename with base_dir) 4) filename and base_dir are relative paths (join under vtr_root) """ # Absolute path if PurePath(filename).is_absolute(): return filename # Under config config_path = Path(config.config_dir) assert config_path.is_absolute() joined_path = config_path / filename if joined_path.exists(): return str(joined_path) # Under base dir base_path = Path(base_dir) if base_path.is_absolute(): # Absolute base joined_path = base_path / filename if joined_path.exists(): return str(joined_path) else: # Relative base under the VTR flow directory joined_path = paths.vtr_flow_path / base_dir / filename if joined_path.exists(): return str(joined_path) # Not found raise InspectError("Failed to resolve VTR source file {}".format(filename))
def check_two_files( config, first_results_filepath, second_results_filepath, first_name="task", second_name="golden", ): """ Compare two files results """ first_results = load_parse_results(first_results_filepath) second_results = load_parse_results(second_results_filepath) # Verify that the architecture and circuit are specified for param in ["architecture", "circuit", "script_params"]: if param not in first_results.PRIMARY_KEYS: raise InspectError( "Required param '{}' missing from {} results: {}".format( param, first_name, first_results_filepath ), first_results_filepath, ) if param not in second_results.PRIMARY_KEYS: raise InspectError( "Required param '{}' missing from {} results: {}".format( param, second_name, second_results_filepath ), second_results_filepath, ) # Verify that all params and pass requirement metric are included in both the result files # We do not worry about non-pass_requriements elements being different or missing pass_req_filepath = str(paths.pass_requirements_path / config.pass_requirements_file) pass_requirements = load_pass_requirements(pass_req_filepath) for metric in pass_requirements.keys(): for ((arch, circuit, script_params), result,) in first_results.all_metrics().items(): if metric not in result: raise InspectError( "Required metric '{}' missing from {} results".format(metric, first_name), first_results_filepath, ) for ((arch, circuit, script_params), result,) in second_results.all_metrics().items(): if metric not in result: raise InspectError( "Required metric '{}' missing from {} results".format(metric, second_name), second_results_filepath, ) # Load the primary keys for result files second_primary_keys = [] for (arch, circuit, script_params), _ in second_results.all_metrics().items(): second_primary_keys.append((arch, circuit, script_params)) first_primary_keys = [] for (arch, circuit, script_params), _ in first_results.all_metrics().items(): first_primary_keys.append((arch, circuit, script_params)) # Ensure that first result file has all the second result file cases for arch, circuit, script_params in second_primary_keys: if first_results.metrics(arch, circuit, script_params) is None: raise InspectError( "Required case {}/{} missing from {} results: {}".format( arch, circuit, first_name, first_results_filepath ) ) # Warn about any elements in first result file that are not found in second result file for arch, circuit, script_params in first_primary_keys: if second_results.metrics(arch, circuit, script_params) is None: print( "Warning: {} includes result for {}/{} missing in {} results".format( first_name, arch, circuit, second_name ) ) num_qor_failures = 0 # Verify that the first results pass each metric for all cases in the second results for (arch, circuit, script_params) in second_primary_keys: second_metrics = second_results.metrics(arch, circuit, script_params) first_metrics = first_results.metrics(arch, circuit, script_params) first_fail = True for metric in pass_requirements.keys(): if not metric in second_metrics: print("Warning: Metric {} missing from {} results".format(metric, second_name)) continue if not metric in first_metrics: print("Warning: Metric {} missing from {} results".format(metric, first_name)) continue try: metric_passed, reason = pass_requirements[metric].check_passed( second_metrics[metric], first_metrics[metric], second_name ) except InspectError as error: metric_passed = False reason = error.msg if not metric_passed: if first_fail: print( "\n{}...[Fail]".format( "/".join(str((Path(config.config_dir).parent)).split("/")[-3:]) ) ) first_fail = False print("[Fail]\n{}/{}/{} {} {}".format(arch, circuit, script_params, metric, reason)) num_qor_failures += 1 return num_qor_failures