def add_to_script(self, script):
script.add_import(imports=["Comparator"], frm="polygraphy.comparator")
RESULTS_VAR_NAME = inline(safe("results"))
comparator_run = make_invocable(
"Comparator.run",
script.get_runners(),
warm_up=self.warm_up,
data_loader=self.data_loader_args.add_to_script(script),
use_subprocess=self.use_subprocess,
save_inputs_path=self.save_inputs)
script.append_suffix(
safe("\n# Runner Execution\n{results} = {:}",
comparator_run,
results=RESULTS_VAR_NAME))
if self.save_results:
G_LOGGER.verbose("Will save runner results to: {:}".format(
self.save_results))
script.add_import(imports=["util"], frm="polygraphy")
script.append_suffix(
safe("\n# Save results\n{results}.save({:})",
self.save_results,
results=RESULTS_VAR_NAME))
return RESULTS_VAR_NAME
def add_to_script(self, script):
# Always required since it is used to print the exit message.
script.append_preimport(safe("from polygraphy.logger import G_LOGGER"))
logger_settings = []
if self.verbosity_count >= 4:
logger_settings.append(
"G_LOGGER.severity = G_LOGGER.ULTRA_VERBOSE")
elif self.verbosity_count == 3:
logger_settings.append(
"G_LOGGER.severity = G_LOGGER.SUPER_VERBOSE")
elif self.verbosity_count == 2:
logger_settings.append(
"G_LOGGER.severity = G_LOGGER.EXTRA_VERBOSE")
elif self.verbosity_count == 1:
logger_settings.append("G_LOGGER.severity = G_LOGGER.VERBOSE")
elif self.verbosity_count == -1:
logger_settings.append("G_LOGGER.severity = G_LOGGER.START")
elif self.verbosity_count == -2:
logger_settings.append("G_LOGGER.severity = G_LOGGER.FINISH")
elif self.verbosity_count == -3:
logger_settings.append("G_LOGGER.severity = G_LOGGER.WARNING")
elif self.verbosity_count == -4:
logger_settings.append("G_LOGGER.severity = G_LOGGER.ERROR")
elif self.verbosity_count <= -4:
logger_settings.append("G_LOGGER.severity = G_LOGGER.CRITICAL")
if self.silent:
logger_settings.append("G_LOGGER.severity = G_LOGGER.CRITICAL")
for fmt in self.log_format:
if fmt == "no-colors":
logger_settings.append("G_LOGGER.colors = False")
elif fmt == "timestamp":
logger_settings.append("G_LOGGER.timestamp = True")
elif fmt == "line-info":
logger_settings.append("G_LOGGER.line_info = True")
if self.log_file:
logger_settings.append("G_LOGGER.log_file = {:}".format(
repr(self.log_file)))
for setting in logger_settings:
script.append_preimport(safe(setting))
return safe("G_LOGGER")
def build_script(self, args):
script = Script(
summary=generate_summary(self.arg_groups[ModelArgs].model_file,
args.runners, args.load_results))
self.arg_groups[LoggerArgs].add_to_script(script)
if not args.runners:
G_LOGGER.warning(
"No runners have been selected. Inference will not be run!")
for runner_arg in args.runners:
add_runner_func = {
"tf": self.arg_groups[TfRunnerArgs].add_to_script,
"onnxrt": self.arg_groups[OnnxrtRunnerArgs].add_to_script,
"trt": self.arg_groups[TrtRunnerArgs].add_to_script,
"trt_legacy": self.arg_groups[TrtLegacyArgs].add_to_script,
"pluginref": self.arg_groups[PluginRefArgs].add_to_script,
}[runner_arg]
add_runner_func(script)
RESULTS_VAR_NAME = self.arg_groups[ComparatorRunArgs].add_to_script(
script)
SUCCESS_VAR_NAME = self.arg_groups[
ComparatorCompareArgs].add_to_script(script,
results_name=RESULTS_VAR_NAME)
script.add_import(imports=["sys"])
cmd_run = inline(safe("' '.join(sys.argv)"))
exit_status = safe(
"# Report Results\n"
"cmd_run = {cmd}\n"
"if not {success}:\n"
'\tG_LOGGER.critical("FAILED | Command: {{}}".format(cmd_run))\n'
'G_LOGGER.finish("PASSED | Command: {{}}".format(cmd_run))\n',
cmd=cmd_run,
success=SUCCESS_VAR_NAME,
)
script.append_suffix(exit_status)
return script
def parse(self, args):
self.trt_outputs = args_util.get_outputs(args, "trt_outputs")
self.caffe_model = args_util.get(args, "caffe_model")
self.batch_size = args_util.get(args, "batch_size")
self.save_uff = args_util.get(args, "save_uff")
self.uff_order = args_util.get(args, "uff_order")
self.preprocessor = args_util.get(args, "preprocessor")
self.calibration_cache = args_util.get(args, "calibration_cache")
calib_base = args_util.get(args, "calibration_base_class")
self.calibration_base_class = None
if calib_base is not None:
calib_base = safe(assert_identifier(calib_base))
self.calibration_base_class = inline(safe("trt.{:}", inline(calib_base)))
self.quantile = args_util.get(args, "quantile")
self.regression_cutoff = args_util.get(args, "regression_cutoff")
self.use_dla = args_util.get(args, "use_dla")
self.allow_gpu_fallback = args_util.get(args, "allow_gpu_fallback")
def run(self, args):
script = Script(
summary=
"Defines or modifies a TensorRT Network using the Network API.",
always_create_runners=False)
script.add_import(imports=["func"], frm="polygraphy")
script.add_import(imports=["tensorrt as trt"])
if self.arg_groups[ModelArgs].model_file is not None:
loader_name = self.arg_groups[
TrtNetworkLoaderArgs].add_trt_network_loader(script)
params = safe("builder, network, parser")
else:
script.add_import(imports=["CreateNetwork"],
frm="polygraphy.backend.trt")
loader_name = safe("CreateNetwork()")
params = safe("builder, network")
script.append_suffix(safe("@func.extend({:})", inline(loader_name)))
script.append_suffix(safe("def load_network({:}):", inline(params)))
script.append_suffix(
safe(
"\tpass # TODO: Set up the network here. This function should not return anything."
))
script.save(args.output)
def run(self, args):
script = Script(summary="Creates a TensorRT Builder Configuration.",
always_create_runners=False)
script.add_import(imports=["func"], frm="polygraphy")
script.add_import(imports=["tensorrt as trt"])
loader_name = self.arg_groups[TrtConfigArgs].add_trt_config_loader(
script)
if not loader_name:
script.add_import(imports=["CreateConfig"],
frm="polygraphy.backend.trt")
loader_name = script.add_loader(safe("CreateConfig()"),
"create_trt_config")
params = safe("config")
script.append_suffix(safe("@func.extend({:})", inline(loader_name)))
script.append_suffix(safe("def load_config({:}):", inline(params)))
script.append_suffix(
safe(
"\tpass # TODO: Set up the builder configuration here. This function should not return anything."
))
script.save(args.output)
def _add_to_script(self, script, user_input_metadata_str=None):
needs_invoke = False
using_random_data = False
if self.data_loader_script:
script.add_import(imports=["mod"], frm="polygraphy")
data_loader = make_invocable("mod.import_from_script",
self.data_loader_script,
name=self.data_loader_func_name)
needs_invoke = True
elif self.load_inputs:
script.add_import(imports=["load_json"], frm="polygraphy.json")
data_loader = safe(
"[]\nfor input_data_path in {load_inputs}:"
"\n\t{data_loader}.extend(load_json(input_data_path, description='input data'))",
load_inputs=self.load_inputs,
data_loader=Script.DATA_LOADER_NAME,
)
else:
using_random_data = True
if user_input_metadata_str is None and self.model_args is not None and self.model_args.input_shapes:
user_input_metadata_str = self.model_args.input_shapes
if user_input_metadata_str:
script.add_import(imports=["TensorMetadata"],
frm="polygraphy.common")
data_loader = make_invocable_if_nondefault(
"DataLoader",
seed=self.seed,
iterations=self.iterations,
input_metadata=user_input_metadata_str,
int_range=self.int_range,
float_range=self.float_range,
val_range=self.val_range,
)
if data_loader:
script.add_import(imports=["DataLoader"],
frm="polygraphy.comparator")
if using_random_data != self.is_using_random_data():
G_LOGGER.internal_error(
"is_using_random_data() reported a false positive!")
return script.set_data_loader(data_loader), needs_invoke
def add_to_script(self, script, user_input_metadata_str=None):
"""
Adds a DataLoader to the script.
Args:
user_input_metadata_str (str(TensorMetadata)):
The name of a variable containing TensorMetadata.
This will control the shape and data type of the generated
data.
"""
if self.data_loader_script:
script.add_import(imports=["invoke_from_script"],
frm="polygraphy.backend.common")
data_loader = make_invocable("invoke_from_script",
self.data_loader_script,
name=self.data_loader_func_name)
elif self.load_inputs:
script.add_import(imports=["load_json"], frm="polygraphy.json")
data_loader = safe(
"[]\nfor input_data_path in {load_inputs}:"
"\n\t{data_loader}.extend(load_json(input_data_path, description='input data'))",
load_inputs=self.load_inputs,
data_loader=Script.DATA_LOADER_NAME)
else:
if user_input_metadata_str is None and self.model_args is not None and self.model_args.input_shapes:
user_input_metadata_str = self.model_args.input_shapes
if user_input_metadata_str:
script.add_import(imports=["TensorMetadata"],
frm="polygraphy.common")
data_loader = make_invocable_if_nondefault(
"DataLoader",
seed=self.seed,
iterations=self.iterations,
input_metadata=user_input_metadata_str,
int_range=self.int_range,
float_range=self.float_range,
val_range=self.val_range)
if data_loader:
script.add_import(imports=["DataLoader"],
frm="polygraphy.comparator")
return script.set_data_loader(data_loader)
def run_script(script_func, *args):
"""
Populates a script using the provided callable, then returns
the variable indicated by the return value of the callable.
Args:
script_func (Callable(Script, *args) -> str):
A callable that populates a Script and then returns
the name of an object defined within the script to retrieve.
args:
Additional positional argruments to pass to script_func.
The script_func should accept these by variable name instead
of taking the values themselves. Values of ``None`` will be
passed directly instead of by variable name.
Returns:
object:
An object defined within the script, or ``None`` if it is not
defined by the script.
"""
script = Script()
arg_names = []
for index, arg in enumerate(args):
if arg is not None:
arg_name = safe("__arg{:}", index)
locals()[arg_name.unwrap()] = arg
arg_names.append(inline(arg_name))
else:
arg_names.append(None)
safe_ret_name = script_func(script, *arg_names)
exec(str(script), globals(), locals())
if safe_ret_name is not None:
ret_name = ensure_safe(safe_ret_name).unwrap()
if ret_name in locals():
return locals()[ret_name]
return None
def script_add(script, arg0=0, arg1=0):
result_name = safe("result")
script.append_suffix(
safe("{:} = {:} + {:}", inline(result_name), arg0, arg1))
return result_name
def get_outputs_for_script(script, outputs):
if outputs == constants.MARK_ALL:
script.add_import(["constants"], frm="polygraphy")
outputs = inline(safe("constants.MARK_ALL"))
return outputs
def add_to_script(self, script, results_name):
script.add_import(imports=["Comparator"], frm="polygraphy.comparator")
if self.load_results:
script.add_import(imports=["util"], frm="polygraphy")
script.add_import(imports=["RunResults"],
frm="polygraphy.comparator")
script.append_suffix(
safe(
"\n# Load results\nfor load_output in {:}:\n\t{results}.extend(RunResults.load(load_output))",
self.load_results,
results=results_name))
if self.top_k is not None:
script.add_import(imports=["PostprocessFunc"],
frm="polygraphy.comparator")
script.append_suffix(
safe(
"\n# Postprocessing - Apply Top-{top_k}\n"
"{results} = Comparator.postprocess({results}, PostprocessFunc.topk_func(k={top_k}))",
top_k=self.top_k,
results=results_name))
SUCCESS_VAR_NAME = inline(safe("success"))
script.append_suffix(
safe("\n{success} = True", success=SUCCESS_VAR_NAME))
if len(
self.runners
) > 1 or self.load_results: # Only do comparisons if there's actually something to compare.
script.append_suffix(safe("# Accuracy Comparison"))
compare_func_str = make_invocable_if_nondefault(
"CompareFunc.basic_compare_func",
rtol=self.rtol,
atol=self.atol,
check_shapes=False if self.no_shape_check else None,
fail_fast=self.fail_fast,
check_error_stat=self.check_error_stat)
compare_func = None
if compare_func_str:
script.add_import(imports=["CompareFunc"],
frm="polygraphy.comparator")
compare_func = inline(safe("compare_func"))
script.append_suffix(
safe("{:} = {:}", compare_func, compare_func_str))
compare_accuracy = make_invocable("Comparator.compare_accuracy",
results_name,
compare_func=compare_func,
fail_fast=self.fail_fast)
script.append_suffix(
safe("{success} &= bool({:})\n",
compare_accuracy,
success=SUCCESS_VAR_NAME))
if self.validate:
script.append_suffix(
safe(
"# Validation\n{success} &= Comparator.validate({results}, check_inf=True, check_nan=True)\n",
success=SUCCESS_VAR_NAME,
results=results_name))
return SUCCESS_VAR_NAME
def add_trt_config_loader(self, script):
profiles = []
for (min_shape, opt_shape, max_shape) in self.profile_dicts:
profile_str = "Profile()"
for name in min_shape.keys():
profile_str += safe(".add({:}, min={:}, opt={:}, max={:})",
name, min_shape[name], opt_shape[name],
max_shape[name]).unwrap()
profiles.append(profile_str)
if profiles:
script.add_import(imports=["Profile"],
frm="polygraphy.backend.trt")
profiles = safe("[\n\t{:}\n]",
inline(safe(",\n\t".join(profiles))))
profile_name = script.add_loader(profiles, "profiles")
else:
profile_name = None
calibrator = None
if any(arg is not None for arg in [
self.calibration_cache, self.calibration_base_class
]) and not self.int8:
G_LOGGER.warning(
"Some int8 calibrator options were set, but int8 precision is not enabled. "
"Calibration options will be ignored. Please set --int8 to enable calibration. "
)
if self.int8 and self.data_loader_args is not None: # We cannot do calibration if there is no data loader.
script.add_import(imports=["Calibrator"],
frm="polygraphy.backend.trt")
script.add_import(imports=["DataLoader"],
frm="polygraphy.comparator")
data_loader_name = self.data_loader_args.add_data_loader(script)
if self.calibration_base_class:
script.add_import(imports=["tensorrt as trt"])
calibrator = make_invocable(
"Calibrator",
data_loader=data_loader_name if data_loader_name else inline(
safe("DataLoader()")),
cache=self.calibration_cache,
BaseClass=self.calibration_base_class,
quantile=self.quantile,
regression_cutoff=self.regression_cutoff,
)
algo_selector = None
if self.load_tactics is not None:
script.add_import(imports=["TacticReplayer"],
frm="polygraphy.backend.trt")
algo_selector = make_invocable("TacticReplayer",
replay=self.load_tactics)
elif self.save_tactics is not None:
script.add_import(imports=["TacticRecorder"],
frm="polygraphy.backend.trt")
algo_selector = make_invocable("TacticRecorder",
record=self.save_tactics)
if self.tactic_sources is not None:
script.add_import(imports=["tensorrt as trt"])
if self.trt_config_script is not None:
script.add_import(imports=["InvokeFromScript"],
frm="polygraphy.backend.common")
config_loader_str = make_invocable("InvokeFromScript",
self.trt_config_script,
name=self.trt_config_func_name)
else:
config_loader_str = make_invocable_if_nondefault(
"CreateTrtConfig",
max_workspace_size=self.workspace,
tf32=self.tf32,
fp16=self.fp16,
int8=self.int8,
strict_types=self.strict_types,
restricted=self.restricted,
profiles=profile_name,
calibrator=calibrator,
load_timing_cache=(self.timing_cache if self.timing_cache
and os.path.exists(self.timing_cache) else
None),
algorithm_selector=algo_selector,
sparse_weights=self.sparse_weights,
tactic_sources=self.tactic_sources,
)
if config_loader_str is not None:
script.add_import(imports=["CreateConfig as CreateTrtConfig"],
frm="polygraphy.backend.trt")
if config_loader_str is not None:
config_loader_name = script.add_loader(config_loader_str,
"create_trt_config")
else:
config_loader_name = None
return config_loader_name
def parse(self, args):
trt_min_shapes = util.default(args_util.get(args, "trt_min_shapes"),
[])
trt_max_shapes = util.default(args_util.get(args, "trt_max_shapes"),
[])
trt_opt_shapes = util.default(args_util.get(args, "trt_opt_shapes"),
[])
default_shapes = TensorMetadata()
if self.model_args is not None:
assert hasattr(self.model_args, "input_shapes"
), "ModelArgs must be parsed before TrtConfigArgs!"
default_shapes = self.model_args.input_shapes
self.profile_dicts = parse_profile_shapes(default_shapes,
trt_min_shapes,
trt_opt_shapes,
trt_max_shapes)
workspace = args_util.get(args, "workspace")
self.workspace = int(workspace) if workspace is not None else workspace
self.tf32 = args_util.get(args, "tf32")
self.fp16 = args_util.get(args, "fp16")
self.int8 = args_util.get(args, "int8")
self.strict_types = args_util.get(args, "strict_types")
self.restricted = args_util.get(args, "restricted")
self.calibration_cache = args_util.get(args, "calibration_cache")
calib_base = args_util.get(args, "calibration_base_class")
self.calibration_base_class = None
if calib_base is not None:
calib_base = safe(assert_identifier(calib_base))
self.calibration_base_class = inline(
safe("trt.{:}", inline(calib_base)))
self.quantile = args_util.get(args, "quantile")
self.regression_cutoff = args_util.get(args, "regression_cutoff")
self.sparse_weights = args_util.get(args, "sparse_weights")
self.timing_cache = args_util.get(args, "timing_cache")
tactic_replay = args_util.get(args, "tactic_replay")
self.load_tactics = args_util.get(args, "load_tactics")
self.save_tactics = args_util.get(args, "save_tactics")
if tactic_replay is not None:
mod.warn_deprecated("--tactic-replay",
"--save-tactics or --load-tactics",
remove_in="0.35.0")
G_LOGGER.warning(
"--tactic-replay is deprecated. Use either --save-tactics or --load-tactics instead."
)
if os.path.exists(
tactic_replay) and util.get_file_size(tactic_replay) > 0:
self.load_tactics = tactic_replay
else:
self.save_tactics = tactic_replay
tactic_sources = args_util.get(args, "tactic_sources")
self.tactic_sources = None
if tactic_sources is not None:
self.tactic_sources = []
for source in tactic_sources:
source = safe(assert_identifier(source.upper()))
source_str = safe("trt.TacticSource.{:}", inline(source))
self.tactic_sources.append(inline(source_str))
self.trt_config_script = args_util.get(args, "trt_config_script")
self.trt_config_func_name = args_util.get(args, "trt_config_func_name")
def add_to_script(self, script):
script.add_import(imports=["TrtLegacyRunner"], frm="polygraphy.backend.trt_legacy")
G_LOGGER.warning("Legacy TensorRT runner only supports implicit batch TensorFlow/UFF, ONNX, and Caffe models")
load_engine = self.model_args.model_file if self.model_args.model_type == "engine" else None
loader_name = None
if self.model_args.model_type == "onnx":
script.add_import(imports=["ParseNetworkFromOnnxLegacy"], frm="polygraphy.backend.trt_legacy")
onnx_loader = self.onnx_loader_args.add_onnx_loader(script, disable_custom_outputs=True)
loader_name = script.add_loader(
make_invocable("ParseNetworkFromOnnxLegacy", onnx_loader), "parse_network_from_onnx_legacy"
)
elif self.model_args.model_type == "caffe":
script.add_import(imports=["LoadNetworkFromCaffe"], frm="polygraphy.backend.trt_legacy")
loader_name = script.add_loader(
make_invocable(
"LoadNetworkFromCaffe",
self.model_args.model_file,
self.caffe_model,
self.trt_outputs,
self.batch_size,
),
"parse_network_from_caffe",
)
elif load_engine is None:
script.add_import(imports=["LoadNetworkFromUff"], frm="polygraphy.backend.trt_legacy")
if self.model_args.model_type == "uff":
script.add_import(imports=["LoadUffFile"], frm="polygraphy.backend.trt_legacy")
shapes = {name: shape for name, (_, shape) in self.model_args.input_shapes.items()}
loader_name = script.add_loader(
make_invocable(
"LoadUffFile", self.model_args.model_file, util.default(shapes, {}), self.trt_outputs
),
"load_uff_file",
)
else:
script.add_import(imports=["ConvertToUff"], frm="polygraphy.backend.trt_legacy")
loader_name = script.add_loader(
make_invocable(
"ConvertToUff",
self.tf_loader_args.add_to_script(script),
save_uff=self.save_uff,
preprocessor=self.preprocessor,
),
"convert_to_uff",
)
loader_name = script.add_loader(
make_invocable("LoadNetworkFromUff", loader_name, uff_order=self.uff_order), "uff_network_loader"
)
calibrator = None
if (
self.trt_config_args.int8 and self.data_loader_args is not None
): # We cannot do calibration if there is no data loader.
script.add_import(imports=["Calibrator"], frm="polygraphy.backend.trt")
script.add_import(imports=["DataLoader"], frm="polygraphy.comparator")
data_loader_name = self.data_loader_args.add_data_loader(script)
if self.calibration_base_class:
script.add_import(imports=["tensorrt as trt"])
calibrator = make_invocable(
"Calibrator",
data_loader=data_loader_name if data_loader_name else inline(safe("DataLoader()")),
cache=self.calibration_cache,
BaseClass=self.calibration_base_class,
quantile=self.quantile,
regression_cutoff=self.regression_cutoff,
)
runner_str = make_invocable(
"TrtLegacyRunner",
network_loader=loader_name,
max_workspace_size=self.trt_config_args.workspace,
max_batch_size=self.batch_size,
fp16=self.trt_config_args.fp16,
tf32=self.trt_config_args.tf32,
load_engine=load_engine,
save_engine=self.trt_engine_save_args.path,
layerwise=self.trt_outputs == constants.MARK_ALL,
plugins=self.trt_engine_loader_args.plugins,
int8=self.trt_config_args.int8,
calibrator=calibrator,
use_dla=self.use_dla,
allow_gpu_fallback=self.allow_gpu_fallback,
)
script.add_runner(runner_str)
