def user_verify_changes(path_A, path_B):
"""
Asks users to verify the changes to the file.
Args:
path_A: Path to a temp file containing the original state of the file before changes
path_B: Path to the file containing the state of the file post-changes. Assumed to be the actual config.json in
configs/
"""
# Assume that path_A, the file that was already checked into the repo, exists and is well formed
with open(path_A) as f:
config_A = json.load(f)
# This inheritently checks if path_B is well-formed JSON. If it is not well formed, this will throw a
# JSONDecodeError with meaningful location on where the syntax error in the JSON.
with open(path_B) as f:
config_B = json.load(f)
changes = find_changes(config_A, config_B)
affected_sys_ids = get_affected_configs(config_A, changes)
if len(affected_sys_ids) == 0:
print("No changes detected")
return
print("Changes in {} will affect the following system IDs:".format(path_B))
for system_id in affected_sys_ids:
print(system_id + ":")
old_conf = get_system_benchmark_config(config_A,
system_id,
enforce_type_equivalence=False)
new_conf = get_system_benchmark_config(config_B,
system_id,
enforce_type_equivalence=False)
print_diffs(affected_sys_ids[system_id],
old_conf,
new_conf,
indent_level=1)
if not prompt_user("Confirm that these changes are expected [y/n]: "):
raise RuntimeError("Aborted by user.")
print("\n============================\n")
def get_affected_configs(config, changes):
"""
Returns a list of system_ids affected by the diff in changes. Changes should be in the format:
{
<system id>: {
<changed key>: <Change type>,
...
}
}
The config provided is the state of config.json *BEFORE* changes are applied.
<Change type> should be of type Change, representing the type of change that was done on
A system_id is considered affected if the provided changes causes the final output of
config_parser.get_system_benchmark_config to be different.
"""
G = DependencyGraph(create_from=config)
# candidates is all system_ids that are potentially impacted by changes
candidates = []
changed_system_ids = dict()
for system_id in changes:
# If a system is an addition or removal, automatically add to changed_system_ids
if type(changes[system_id]) is Change:
assert changes[system_id] in (Change.Addition, Change.Removal)
changed_system_ids[system_id] = changes[system_id]
else:
candidates += G.BFS_from(system_id)
candidates = list(set(candidates)) # Filter out duplicates
changed_config = copy.deepcopy(config)
update_nested(changed_config, changes)
for candidate in candidates:
new_conf = get_system_benchmark_config(changed_config,
candidate,
enforce_type_equivalence=False)
found_changes = extract_changes(new_conf)
if len(found_changes) > 0:
changed_system_ids[candidate] = found_changes
return changed_system_ids
def main(main_args, system):
"""
Args:
main_args: Args parsed from user input.
system: System to use
"""
system_id = system.get_id()
# Turn off MPS in case it's turned on.
turn_off_mps()
# Get user's benchmarks, else run all.
benchmarks = BENCHMARKS.ALL
if main_args["benchmarks"] is not None:
benchmarks = main_args["benchmarks"].split(",")
benchmarks = [BENCHMARKS.alias(b) for b in benchmarks]
# Get user's scenarios, else use all.
scenarios = SCENARIOS.ALL
if main_args["scenarios"] is not None:
scenarios = main_args["scenarios"].split(",")
scenarios = [SCENARIOS.alias(s) for s in scenarios]
profile = main_args.get("profile", None)
power = main_args.get("power", False)
# Automatically find config file paths
config_files = main_args["configs"]
if config_files == "" or config_files is None:
config_files = find_config_files(benchmarks, scenarios)
if config_files == "":
logging.warn(
"Cannot find any valid configs for the specified benchmark-scenario pairs."
)
return
logging.info("Using config files: {:}".format(str(config_files)))
configs = load_configs(config_files)
for config in configs:
base_benchmark_conf = get_system_benchmark_config(config, system_id)
if base_benchmark_conf is None:
continue
base_benchmark_conf["config_name"] = "{:}_{:}_{:}".format(
system_id, base_benchmark_conf["benchmark"],
base_benchmark_conf["scenario"])
logging.info("Processing config \"{:}\"".format(
base_benchmark_conf["config_name"]))
# Load config_ver / apply overrides
conf_vers = main_args.get("config_ver", "default").split(",")
# Build default first. This is because some config_vers only modify harness args, and the engine is the same as
# default. In this case, we build default first, and copy it instead of rebuilding it.
if "default" in conf_vers:
conf_vers = ["default"] + list(set(conf_vers) - {"default"})
elif "all" in conf_vers:
conf_vers = ["default"] + list(
base_benchmark_conf.get("config_ver", {}).keys())
for conf_ver in conf_vers:
benchmark_conf = dict(
base_benchmark_conf) # Copy the config so we don't modify it
# These fields are canonical names that refer to certain config versions
benchmark_conf["accuracy_level"] = "99%"
benchmark_conf["optimization_level"] = "plugin-enabled"
benchmark_conf["inference_server"] = "lwis"
"""@etcheng
NOTE: The original plan was to use a syntax like high_accuracy+triton to be able to combine already defined
config_vers. However, since high_accuracy, triton, and high_accuracy+triton are likely to all have different
expected QPS values, it makes more sense to keep high_accuracy_triton as a separate, individual config_ver.
In the future, perhaps we can make an "extends": [ list of strings ] or { dict of config_ver name ->
config_key } field in config_vers, so that we can define new config_vers that extend or combine previous
config_vers.
"""
equiv_to_default = False
if conf_ver != "default":
if "config_ver" not in benchmark_conf or conf_ver not in benchmark_conf[
"config_ver"]:
logging.warn(
"--config_ver={:} does not exist in config file '{:}'".
format(conf_ver, benchmark_conf["config_name"]))
continue
else:
if "high_accuracy" in conf_ver:
benchmark_conf["accuracy_level"] = "99.9%"
if "ootb" in conf_ver:
benchmark_conf["optimization_level"] = "ootb"
# "inference_server" is set when we run the harness
overrides = benchmark_conf["config_ver"][conf_ver]
# Check if this config_ver is equivalent to the default engine
gen_eng_argset = set(common_args.GENERATE_ENGINE_ARGS)
override_argset = set(overrides.keys())
equiv_to_default = (len(gen_eng_argset
& override_argset) == 0)
benchmark_conf.update(overrides)
# Update the config_ver key to be the actual string name, not the overrides
benchmark_conf["config_ver"] = conf_ver
need_gpu = not main_args["no_gpu"]
need_dla = not main_args["gpu_only"]
# Override the system_name if it exists
if "system_name" in main_args:
benchmark_conf["system_name"] = main_args["system_name"]
# Generate engines.
if main_args["action"] == "generate_engines":
# Turn on MPS if server scenario and if active_sms is specified.
benchmark_conf = apply_overrides(benchmark_conf,
["active_sms"])
active_sms = benchmark_conf.get("active_sms", None)
copy_from_default = ("default"
in conf_vers) and equiv_to_default
if copy_from_default:
logging.info(
"config_ver={:} only modifies harness args. Re-using default engine."
.format(conf_ver))
_gen_args = [benchmark_conf]
_gen_kwargs = {
"gpu": need_gpu,
"dla": need_dla,
"copy_from_default": copy_from_default
}
if not main_args["no_child_process"]:
if config[
"scenario"] == SCENARIOS.Server and active_sms is not None and active_sms < 100:
with ScopedMPS(active_sms):
launch_handle_generate_engine(
*_gen_args, **_gen_kwargs)
else:
launch_handle_generate_engine(*_gen_args,
**_gen_kwargs)
else:
handle_generate_engine(*_gen_args, **_gen_kwargs)
# Run harness.
elif main_args["action"] == "run_harness":
# In case there's a leftover audit.config file from a prior compliance run or other reason
# we need to delete it or we risk silent failure.
auditing.cleanup()
handle_run_harness(benchmark_conf, need_gpu, need_dla, profile,
power)
elif main_args["action"] == "run_audit_harness":
logging.info('\n\n\nRunning compliance harness for test ' +
main_args['audit_test'] + '\n\n\n')
# Find the correct audit.config file and move it in current directory
dest_config = auditing.load(main_args['audit_test'],
benchmark_conf['benchmark'])
# Make sure the log_file override is valid
os.makedirs("build/compliance_logs", exist_ok=True)
# Pass audit test name to handle_run_harness via benchmark_conf
benchmark_conf['audit_test_name'] = main_args['audit_test']
# Run harness
handle_run_harness(benchmark_conf,
need_gpu,
need_dla,
profile,
power,
compliance=True)
# Cleanup audit.config
logging.info("AUDIT HARNESS: Cleaning Up audit.config...")
auditing.cleanup()
elif main_args["action"] == "run_audit_verification":
logging.info("Running compliance verification for test " +
main_args['audit_test'])
handle_audit_verification(
audit_test_name=main_args['audit_test'],
config=benchmark_conf)
auditing.cleanup()
elif main_args["action"] == "calibrate":
# To generate calibration cache, we only need to run each benchmark once.
# Use offline config.
if benchmark_conf["scenario"] == SCENARIOS.Offline:
handle_calibrate(benchmark_conf)
elif main_args["action"] == "generate_conf_files":
handle_run_harness(benchmark_conf,
need_gpu,
need_dla,
generate_conf_files_only=True)
def __init__(self, bench, scen, config_dict, config_funcs=None):
""" Construct a ConfigGrid
Args:
bench (str): The benchmark requested (fuzzy match behavior using BENCHMARKS.alias)
scen (str): The scenario requested (fuzzy match behavior using SCENARIOS.alias)
config_dict (Dict[str, List]): A config dictionary. Refer to 'Config Schema' in the README for format
config_funcs (Dict[str, Callable]): A dictionary of META* functions. Refer to 'Config Schema' in the README for requirements.
"""
if args.spoof_system_id:
self.system_id = args.spoof_system_id
else:
self.system = get_system()
self.system_id = self.system.get_id()
self.benchmark = BENCHMARKS.alias(bench)
self.scenario = SCENARIOS.alias(scen)
candidate_configs = find_config_files(benchmarks=[self.benchmark],
scenarios=[self.scenario])
configs = load_configs(candidate_configs)
assert len(configs) == 1
# To work with "extends" and "scales", we need to call into another config helper:
self.base_config = get_system_benchmark_config(configs[0], self.system_id)
self.default_config = configs[0]['default']
griddict = config_dict
self.no_rebuild_params = None
# No-op
self.is_config_valid = lambda x: True
# No-op
self.search_callback = None
self.replay = None
funcs_processed = set()
if config_funcs:
if config_funcs.get("META_search_callback"):
funcs_processed.add("META_search_callback")
self.search_callback = config_funcs['META_search_callback']
if not args.use_cached:
raise RuntimeError(f"META_search_callback must be used with --use_cached for reproducibility.")
if config_funcs.get("META_get_no_rebuild_params"):
funcs_processed.add("META_get_no_rebuild_params")
norebuild_params = config_funcs.get("META_get_no_rebuild_params")()
assert isinstance(norebuild_params, list)
# Make sure these keys all exist in our grid params:
# But we might not know grid params if a search_callback is being used:
if self.search_callback is None:
missing_keys = set(norebuild_params) - set(griddict.keys())
if len(missing_keys) > 0:
raise RuntimeError(f"The keys: {missing_keys} were mentioned in META_get_no_rebuild_params, but are not a specified parameter in:\n{griddict.keys()}")
else:
print("WARNING: Not checking get_no_rebuild_params against grid parameters, be careful")
# For use later, we're gonna turn this into a set:
self.no_rebuild_params = set(norebuild_params)
if config_funcs.get("META_is_config_valid"):
funcs_processed.add("META_is_config_valid")
self.is_config_valid = config_funcs["META_is_config_valid"]
# Make sure we aren't scanning other params:
# Other META handling goes here
unmatched_funcs = set(config_funcs.keys()) - funcs_processed
if len(unmatched_funcs) > 0:
raise RuntimeError(f"Found the following META functions which haven't been implemented, refer to README for proper naming {unmatched_funcs}")
# Make sure we can set all keys are in our config:
if not args.no_check_keys:
for grid_key in griddict.keys():
if grid_key not in self.base_config:
print(f"{grid_key} not found in base config")
print(f"{self.base_config}")
assert False
# Make sure all values are non-empty lists of something that isn't a list or a dict
# TODO expand this to something reasonable to help META_bisect
if "META_search_callback" in funcs_processed:
print("WARNING: Skipping parameter validation because META_search_callback was provided")
else:
for val in griddict.values():
assert isinstance(val, list)
#assert len(val) >= 1
assert all(not isinstance(el, list) and not isinstance(el, dict) for el in val)
self.grid = griddict
def __init__(self, bench, scen, config_dict, config_funcs=None):
""" Construct a ConfigGrid
Args:
bench (str): The benchmark requested (fuzzy match behavior using BENCHMARKS.alias)
scen (str): The scenario requested (fuzzy match behavior using SCENARIOS.alias)
config_dict (Dict[str, List]): A config dictionary. Refer to 'Config Schema' in the README for format
config_funcs (Dict[str, Callable]): A dictionary of META* functions. Refer to 'Config Schema' in the README for requirements.
"""
self.system = get_system()
self.system_id = self.system.get_id()
self.benchmark = BENCHMARKS.alias(bench)
self.scenario = SCENARIOS.alias(scen)
candidate_configs = find_config_files(benchmarks=[self.benchmark],
scenarios=[self.scenario])
configs = load_configs(candidate_configs)
assert len(configs) == 1
# To work with "extends" and "scales", we need to call into another config helper:
self.base_config = get_system_benchmark_config(configs[0],
self.system_id)
self.default_config = configs[0]['default']
griddict = config_dict
self.no_rebuild_params = None
# No-op
self.is_config_valid = lambda x: True
funcs_processed = set()
if config_funcs:
if config_funcs.get("META_get_no_rebuild_params"):
funcs_processed.add("META_get_no_rebuild_params")
norebuild_params = config_funcs.get(
"META_get_no_rebuild_params")()
assert isinstance(norebuild_params, list)
# Make sure these keys all exist in our grid params:
missing_keys = set(norebuild_params) - set(griddict.keys())
if len(missing_keys) > 0:
raise RuntimeError(
f"The keys: {missing_keys} were mentioned in META_get_no_rebuild_params, but are not a specified parameter in:\n{griddict.keys()}"
)
# For use later, we're gonna turn this into a set:
self.no_rebuild_params = set(norebuild_params)
if config_funcs.get("META_is_config_valid"):
funcs_processed.add("META_is_config_valid")
self.is_config_valid = config_funcs["META_is_config_valid"]
# Other META handling goes here
unmatched_funcs = set(config_funcs.keys()) - funcs_processed
if len(unmatched_funcs) > 0:
raise RuntimeError(
f"Found the following META functions which haven't been implemented, refer to README for proper naming {unmatched_funcs}"
)
# Make sure we can set all keys are in our config:
for grid_key in griddict.keys():
if grid_key not in self.base_config:
print(f"{grid_key} not found in base config")
print(f"{self.base_config}")
assert False
# Make sure all values are non-empty lists of something that isn't a list or a dict
for val in griddict.values():
assert isinstance(val, list)
assert len(val) >= 1
assert all(not isinstance(el, list) and not isinstance(el, dict)
for el in val)
self.grid = griddict