def add_to_script(self, script):
config_loader_str = make_invocable_if_nondefault(
"CreateConfig",
gpu_memory_fraction=self.gpu_memory_fraction,
allow_growth=self.allow_growth,
use_xla=self.xla)
if config_loader_str is not None:
script.add_import(imports=["CreateConfig"],
frm="polygraphy.backend.tf")
config_loader_name = script.add_loader(config_loader_str,
"create_tf_config")
else:
config_loader_name = None
return config_loader_name
def _add_to_script(self, script, user_input_metadata_str=None):
needs_invoke = False
using_random_data = False
if self.data_loader_script:
script.add_import(imports=["mod"], frm="polygraphy")
data_loader = make_invocable("mod.import_from_script",
self.data_loader_script,
name=self.data_loader_func_name)
needs_invoke = True
elif self.load_inputs:
script.add_import(imports=["load_json"], frm="polygraphy.json")
data_loader = safe(
"[]\nfor input_data_path in {load_inputs}:"
"\n\t{data_loader}.extend(load_json(input_data_path, description='input data'))",
load_inputs=self.load_inputs,
data_loader=Script.DATA_LOADER_NAME,
)
else:
using_random_data = True
if user_input_metadata_str is None and self.model_args is not None and self.model_args.input_shapes:
user_input_metadata_str = self.model_args.input_shapes
if user_input_metadata_str:
script.add_import(imports=["TensorMetadata"],
frm="polygraphy.common")
data_loader = make_invocable_if_nondefault(
"DataLoader",
seed=self.seed,
iterations=self.iterations,
input_metadata=user_input_metadata_str,
int_range=self.int_range,
float_range=self.float_range,
val_range=self.val_range,
)
if data_loader:
script.add_import(imports=["DataLoader"],
frm="polygraphy.comparator")
if using_random_data != self.is_using_random_data():
G_LOGGER.internal_error(
"is_using_random_data() reported a false positive!")
return script.set_data_loader(data_loader), needs_invoke
def add_to_script(self, script, user_input_metadata_str=None):
"""
Adds a DataLoader to the script.
Args:
user_input_metadata_str (str(TensorMetadata)):
The name of a variable containing TensorMetadata.
This will control the shape and data type of the generated
data.
"""
if self.data_loader_script:
script.add_import(imports=["invoke_from_script"],
frm="polygraphy.backend.common")
data_loader = make_invocable("invoke_from_script",
self.data_loader_script,
name=self.data_loader_func_name)
elif self.load_inputs:
script.add_import(imports=["load_json"], frm="polygraphy.json")
data_loader = safe(
"[]\nfor input_data_path in {load_inputs}:"
"\n\t{data_loader}.extend(load_json(input_data_path, description='input data'))",
load_inputs=self.load_inputs,
data_loader=Script.DATA_LOADER_NAME)
else:
if user_input_metadata_str is None and self.model_args is not None and self.model_args.input_shapes:
user_input_metadata_str = self.model_args.input_shapes
if user_input_metadata_str:
script.add_import(imports=["TensorMetadata"],
frm="polygraphy.common")
data_loader = make_invocable_if_nondefault(
"DataLoader",
seed=self.seed,
iterations=self.iterations,
input_metadata=user_input_metadata_str,
int_range=self.int_range,
float_range=self.float_range,
val_range=self.val_range)
if data_loader:
script.add_import(imports=["DataLoader"],
frm="polygraphy.comparator")
return script.set_data_loader(data_loader)
def test_invoke_if_nondefault_none_args(self):
assert make_invocable_if_nondefault("Dummy", None) is None
assert make_invocable_if_nondefault("Dummy", x=None) is None
def add_to_script(self, script, results_name):
script.add_import(imports=["Comparator"], frm="polygraphy.comparator")
if self.load_results:
script.add_import(imports=["util"], frm="polygraphy")
script.add_import(imports=["RunResults"],
frm="polygraphy.comparator")
script.append_suffix(
safe(
"\n# Load results\nfor load_output in {:}:\n\t{results}.extend(RunResults.load(load_output))",
self.load_results,
results=results_name))
if self.top_k is not None:
script.add_import(imports=["PostprocessFunc"],
frm="polygraphy.comparator")
script.append_suffix(
safe(
"\n# Postprocessing - Apply Top-{top_k}\n"
"{results} = Comparator.postprocess({results}, PostprocessFunc.topk_func(k={top_k}))",
top_k=self.top_k,
results=results_name))
SUCCESS_VAR_NAME = inline(safe("success"))
script.append_suffix(
safe("\n{success} = True", success=SUCCESS_VAR_NAME))
if len(
self.runners
) > 1 or self.load_results: # Only do comparisons if there's actually something to compare.
script.append_suffix(safe("# Accuracy Comparison"))
compare_func_str = make_invocable_if_nondefault(
"CompareFunc.basic_compare_func",
rtol=self.rtol,
atol=self.atol,
check_shapes=False if self.no_shape_check else None,
fail_fast=self.fail_fast,
check_error_stat=self.check_error_stat)
compare_func = None
if compare_func_str:
script.add_import(imports=["CompareFunc"],
frm="polygraphy.comparator")
compare_func = inline(safe("compare_func"))
script.append_suffix(
safe("{:} = {:}", compare_func, compare_func_str))
compare_accuracy = make_invocable("Comparator.compare_accuracy",
results_name,
compare_func=compare_func,
fail_fast=self.fail_fast)
script.append_suffix(
safe("{success} &= bool({:})\n",
compare_accuracy,
success=SUCCESS_VAR_NAME))
if self.validate:
script.append_suffix(
safe(
"# Validation\n{success} &= Comparator.validate({results}, check_inf=True, check_nan=True)\n",
success=SUCCESS_VAR_NAME,
results=results_name))
return SUCCESS_VAR_NAME
def add_trt_config_loader(self, script):
profiles = []
for (min_shape, opt_shape, max_shape) in self.profile_dicts:
profile_str = "Profile()"
for name in min_shape.keys():
profile_str += safe(".add({:}, min={:}, opt={:}, max={:})",
name, min_shape[name], opt_shape[name],
max_shape[name]).unwrap()
profiles.append(profile_str)
if profiles:
script.add_import(imports=["Profile"],
frm="polygraphy.backend.trt")
profiles = safe("[\n\t{:}\n]",
inline(safe(",\n\t".join(profiles))))
profile_name = script.add_loader(profiles, "profiles")
else:
profile_name = None
calibrator = None
if any(arg is not None for arg in [
self.calibration_cache, self.calibration_base_class
]) and not self.int8:
G_LOGGER.warning(
"Some int8 calibrator options were set, but int8 precision is not enabled. "
"Calibration options will be ignored. Please set --int8 to enable calibration. "
)
if self.int8 and self.data_loader_args is not None: # We cannot do calibration if there is no data loader.
script.add_import(imports=["Calibrator"],
frm="polygraphy.backend.trt")
script.add_import(imports=["DataLoader"],
frm="polygraphy.comparator")
data_loader_name = self.data_loader_args.add_data_loader(script)
if self.calibration_base_class:
script.add_import(imports=["tensorrt as trt"])
calibrator = make_invocable(
"Calibrator",
data_loader=data_loader_name if data_loader_name else inline(
safe("DataLoader()")),
cache=self.calibration_cache,
BaseClass=self.calibration_base_class,
quantile=self.quantile,
regression_cutoff=self.regression_cutoff,
)
algo_selector = None
if self.load_tactics is not None:
script.add_import(imports=["TacticReplayer"],
frm="polygraphy.backend.trt")
algo_selector = make_invocable("TacticReplayer",
replay=self.load_tactics)
elif self.save_tactics is not None:
script.add_import(imports=["TacticRecorder"],
frm="polygraphy.backend.trt")
algo_selector = make_invocable("TacticRecorder",
record=self.save_tactics)
if self.tactic_sources is not None:
script.add_import(imports=["tensorrt as trt"])
if self.trt_config_script is not None:
script.add_import(imports=["InvokeFromScript"],
frm="polygraphy.backend.common")
config_loader_str = make_invocable("InvokeFromScript",
self.trt_config_script,
name=self.trt_config_func_name)
else:
config_loader_str = make_invocable_if_nondefault(
"CreateTrtConfig",
max_workspace_size=self.workspace,
tf32=self.tf32,
fp16=self.fp16,
int8=self.int8,
strict_types=self.strict_types,
restricted=self.restricted,
profiles=profile_name,
calibrator=calibrator,
load_timing_cache=(self.timing_cache if self.timing_cache
and os.path.exists(self.timing_cache) else
None),
algorithm_selector=algo_selector,
sparse_weights=self.sparse_weights,
tactic_sources=self.tactic_sources,
)
if config_loader_str is not None:
script.add_import(imports=["CreateConfig as CreateTrtConfig"],
frm="polygraphy.backend.trt")
if config_loader_str is not None:
config_loader_name = script.add_loader(config_loader_str,
"create_trt_config")
else:
config_loader_name = None
return config_loader_name