def add_to_script(self, script): script.add_import(imports=["Comparator"], frm="polygraphy.comparator") RESULTS_VAR_NAME = inline(safe("results")) comparator_run = make_invocable( "Comparator.run", script.get_runners(), warm_up=self.warm_up, data_loader=self.data_loader_args.add_to_script(script), use_subprocess=self.use_subprocess, save_inputs_path=self.save_inputs) script.append_suffix( safe("\n# Runner Execution\n{results} = {:}", comparator_run, results=RESULTS_VAR_NAME)) if self.save_results: G_LOGGER.verbose("Will save runner results to: {:}".format( self.save_results)) script.add_import(imports=["util"], frm="polygraphy") script.append_suffix( safe("\n# Save results\n{results}.save({:})", self.save_results, results=RESULTS_VAR_NAME)) return RESULTS_VAR_NAME
def add_to_script(self, script): # Always required since it is used to print the exit message. script.append_preimport(safe("from polygraphy.logger import G_LOGGER")) logger_settings = [] if self.verbosity_count >= 4: logger_settings.append( "G_LOGGER.severity = G_LOGGER.ULTRA_VERBOSE") elif self.verbosity_count == 3: logger_settings.append( "G_LOGGER.severity = G_LOGGER.SUPER_VERBOSE") elif self.verbosity_count == 2: logger_settings.append( "G_LOGGER.severity = G_LOGGER.EXTRA_VERBOSE") elif self.verbosity_count == 1: logger_settings.append("G_LOGGER.severity = G_LOGGER.VERBOSE") elif self.verbosity_count == -1: logger_settings.append("G_LOGGER.severity = G_LOGGER.START") elif self.verbosity_count == -2: logger_settings.append("G_LOGGER.severity = G_LOGGER.FINISH") elif self.verbosity_count == -3: logger_settings.append("G_LOGGER.severity = G_LOGGER.WARNING") elif self.verbosity_count == -4: logger_settings.append("G_LOGGER.severity = G_LOGGER.ERROR") elif self.verbosity_count <= -4: logger_settings.append("G_LOGGER.severity = G_LOGGER.CRITICAL") if self.silent: logger_settings.append("G_LOGGER.severity = G_LOGGER.CRITICAL") for fmt in self.log_format: if fmt == "no-colors": logger_settings.append("G_LOGGER.colors = False") elif fmt == "timestamp": logger_settings.append("G_LOGGER.timestamp = True") elif fmt == "line-info": logger_settings.append("G_LOGGER.line_info = True") if self.log_file: logger_settings.append("G_LOGGER.log_file = {:}".format( repr(self.log_file))) for setting in logger_settings: script.append_preimport(safe(setting)) return safe("G_LOGGER")
def build_script(self, args): script = Script( summary=generate_summary(self.arg_groups[ModelArgs].model_file, args.runners, args.load_results)) self.arg_groups[LoggerArgs].add_to_script(script) if not args.runners: G_LOGGER.warning( "No runners have been selected. Inference will not be run!") for runner_arg in args.runners: add_runner_func = { "tf": self.arg_groups[TfRunnerArgs].add_to_script, "onnxrt": self.arg_groups[OnnxrtRunnerArgs].add_to_script, "trt": self.arg_groups[TrtRunnerArgs].add_to_script, "trt_legacy": self.arg_groups[TrtLegacyArgs].add_to_script, "pluginref": self.arg_groups[PluginRefArgs].add_to_script, }[runner_arg] add_runner_func(script) RESULTS_VAR_NAME = self.arg_groups[ComparatorRunArgs].add_to_script( script) SUCCESS_VAR_NAME = self.arg_groups[ ComparatorCompareArgs].add_to_script(script, results_name=RESULTS_VAR_NAME) script.add_import(imports=["sys"]) cmd_run = inline(safe("' '.join(sys.argv)")) exit_status = safe( "# Report Results\n" "cmd_run = {cmd}\n" "if not {success}:\n" '\tG_LOGGER.critical("FAILED | Command: {{}}".format(cmd_run))\n' 'G_LOGGER.finish("PASSED | Command: {{}}".format(cmd_run))\n', cmd=cmd_run, success=SUCCESS_VAR_NAME, ) script.append_suffix(exit_status) return script
def parse(self, args): self.trt_outputs = args_util.get_outputs(args, "trt_outputs") self.caffe_model = args_util.get(args, "caffe_model") self.batch_size = args_util.get(args, "batch_size") self.save_uff = args_util.get(args, "save_uff") self.uff_order = args_util.get(args, "uff_order") self.preprocessor = args_util.get(args, "preprocessor") self.calibration_cache = args_util.get(args, "calibration_cache") calib_base = args_util.get(args, "calibration_base_class") self.calibration_base_class = None if calib_base is not None: calib_base = safe(assert_identifier(calib_base)) self.calibration_base_class = inline(safe("trt.{:}", inline(calib_base))) self.quantile = args_util.get(args, "quantile") self.regression_cutoff = args_util.get(args, "regression_cutoff") self.use_dla = args_util.get(args, "use_dla") self.allow_gpu_fallback = args_util.get(args, "allow_gpu_fallback")
def run(self, args): script = Script( summary= "Defines or modifies a TensorRT Network using the Network API.", always_create_runners=False) script.add_import(imports=["func"], frm="polygraphy") script.add_import(imports=["tensorrt as trt"]) if self.arg_groups[ModelArgs].model_file is not None: loader_name = self.arg_groups[ TrtNetworkLoaderArgs].add_trt_network_loader(script) params = safe("builder, network, parser") else: script.add_import(imports=["CreateNetwork"], frm="polygraphy.backend.trt") loader_name = safe("CreateNetwork()") params = safe("builder, network") script.append_suffix(safe("@func.extend({:})", inline(loader_name))) script.append_suffix(safe("def load_network({:}):", inline(params))) script.append_suffix( safe( "\tpass # TODO: Set up the network here. This function should not return anything." )) script.save(args.output)
def run(self, args): script = Script(summary="Creates a TensorRT Builder Configuration.", always_create_runners=False) script.add_import(imports=["func"], frm="polygraphy") script.add_import(imports=["tensorrt as trt"]) loader_name = self.arg_groups[TrtConfigArgs].add_trt_config_loader( script) if not loader_name: script.add_import(imports=["CreateConfig"], frm="polygraphy.backend.trt") loader_name = script.add_loader(safe("CreateConfig()"), "create_trt_config") params = safe("config") script.append_suffix(safe("@func.extend({:})", inline(loader_name))) script.append_suffix(safe("def load_config({:}):", inline(params))) script.append_suffix( safe( "\tpass # TODO: Set up the builder configuration here. This function should not return anything." )) script.save(args.output)
def _add_to_script(self, script, user_input_metadata_str=None): needs_invoke = False using_random_data = False if self.data_loader_script: script.add_import(imports=["mod"], frm="polygraphy") data_loader = make_invocable("mod.import_from_script", self.data_loader_script, name=self.data_loader_func_name) needs_invoke = True elif self.load_inputs: script.add_import(imports=["load_json"], frm="polygraphy.json") data_loader = safe( "[]\nfor input_data_path in {load_inputs}:" "\n\t{data_loader}.extend(load_json(input_data_path, description='input data'))", load_inputs=self.load_inputs, data_loader=Script.DATA_LOADER_NAME, ) else: using_random_data = True if user_input_metadata_str is None and self.model_args is not None and self.model_args.input_shapes: user_input_metadata_str = self.model_args.input_shapes if user_input_metadata_str: script.add_import(imports=["TensorMetadata"], frm="polygraphy.common") data_loader = make_invocable_if_nondefault( "DataLoader", seed=self.seed, iterations=self.iterations, input_metadata=user_input_metadata_str, int_range=self.int_range, float_range=self.float_range, val_range=self.val_range, ) if data_loader: script.add_import(imports=["DataLoader"], frm="polygraphy.comparator") if using_random_data != self.is_using_random_data(): G_LOGGER.internal_error( "is_using_random_data() reported a false positive!") return script.set_data_loader(data_loader), needs_invoke
def add_to_script(self, script, user_input_metadata_str=None): """ Adds a DataLoader to the script. Args: user_input_metadata_str (str(TensorMetadata)): The name of a variable containing TensorMetadata. This will control the shape and data type of the generated data. """ if self.data_loader_script: script.add_import(imports=["invoke_from_script"], frm="polygraphy.backend.common") data_loader = make_invocable("invoke_from_script", self.data_loader_script, name=self.data_loader_func_name) elif self.load_inputs: script.add_import(imports=["load_json"], frm="polygraphy.json") data_loader = safe( "[]\nfor input_data_path in {load_inputs}:" "\n\t{data_loader}.extend(load_json(input_data_path, description='input data'))", load_inputs=self.load_inputs, data_loader=Script.DATA_LOADER_NAME) else: if user_input_metadata_str is None and self.model_args is not None and self.model_args.input_shapes: user_input_metadata_str = self.model_args.input_shapes if user_input_metadata_str: script.add_import(imports=["TensorMetadata"], frm="polygraphy.common") data_loader = make_invocable_if_nondefault( "DataLoader", seed=self.seed, iterations=self.iterations, input_metadata=user_input_metadata_str, int_range=self.int_range, float_range=self.float_range, val_range=self.val_range) if data_loader: script.add_import(imports=["DataLoader"], frm="polygraphy.comparator") return script.set_data_loader(data_loader)
def run_script(script_func, *args): """ Populates a script using the provided callable, then returns the variable indicated by the return value of the callable. Args: script_func (Callable(Script, *args) -> str): A callable that populates a Script and then returns the name of an object defined within the script to retrieve. args: Additional positional argruments to pass to script_func. The script_func should accept these by variable name instead of taking the values themselves. Values of ``None`` will be passed directly instead of by variable name. Returns: object: An object defined within the script, or ``None`` if it is not defined by the script. """ script = Script() arg_names = [] for index, arg in enumerate(args): if arg is not None: arg_name = safe("__arg{:}", index) locals()[arg_name.unwrap()] = arg arg_names.append(inline(arg_name)) else: arg_names.append(None) safe_ret_name = script_func(script, *arg_names) exec(str(script), globals(), locals()) if safe_ret_name is not None: ret_name = ensure_safe(safe_ret_name).unwrap() if ret_name in locals(): return locals()[ret_name] return None
def script_add(script, arg0=0, arg1=0): result_name = safe("result") script.append_suffix( safe("{:} = {:} + {:}", inline(result_name), arg0, arg1)) return result_name
def get_outputs_for_script(script, outputs): if outputs == constants.MARK_ALL: script.add_import(["constants"], frm="polygraphy") outputs = inline(safe("constants.MARK_ALL")) return outputs
def add_to_script(self, script, results_name): script.add_import(imports=["Comparator"], frm="polygraphy.comparator") if self.load_results: script.add_import(imports=["util"], frm="polygraphy") script.add_import(imports=["RunResults"], frm="polygraphy.comparator") script.append_suffix( safe( "\n# Load results\nfor load_output in {:}:\n\t{results}.extend(RunResults.load(load_output))", self.load_results, results=results_name)) if self.top_k is not None: script.add_import(imports=["PostprocessFunc"], frm="polygraphy.comparator") script.append_suffix( safe( "\n# Postprocessing - Apply Top-{top_k}\n" "{results} = Comparator.postprocess({results}, PostprocessFunc.topk_func(k={top_k}))", top_k=self.top_k, results=results_name)) SUCCESS_VAR_NAME = inline(safe("success")) script.append_suffix( safe("\n{success} = True", success=SUCCESS_VAR_NAME)) if len( self.runners ) > 1 or self.load_results: # Only do comparisons if there's actually something to compare. script.append_suffix(safe("# Accuracy Comparison")) compare_func_str = make_invocable_if_nondefault( "CompareFunc.basic_compare_func", rtol=self.rtol, atol=self.atol, check_shapes=False if self.no_shape_check else None, fail_fast=self.fail_fast, check_error_stat=self.check_error_stat) compare_func = None if compare_func_str: script.add_import(imports=["CompareFunc"], frm="polygraphy.comparator") compare_func = inline(safe("compare_func")) script.append_suffix( safe("{:} = {:}", compare_func, compare_func_str)) compare_accuracy = make_invocable("Comparator.compare_accuracy", results_name, compare_func=compare_func, fail_fast=self.fail_fast) script.append_suffix( safe("{success} &= bool({:})\n", compare_accuracy, success=SUCCESS_VAR_NAME)) if self.validate: script.append_suffix( safe( "# Validation\n{success} &= Comparator.validate({results}, check_inf=True, check_nan=True)\n", success=SUCCESS_VAR_NAME, results=results_name)) return SUCCESS_VAR_NAME
def add_trt_config_loader(self, script): profiles = [] for (min_shape, opt_shape, max_shape) in self.profile_dicts: profile_str = "Profile()" for name in min_shape.keys(): profile_str += safe(".add({:}, min={:}, opt={:}, max={:})", name, min_shape[name], opt_shape[name], max_shape[name]).unwrap() profiles.append(profile_str) if profiles: script.add_import(imports=["Profile"], frm="polygraphy.backend.trt") profiles = safe("[\n\t{:}\n]", inline(safe(",\n\t".join(profiles)))) profile_name = script.add_loader(profiles, "profiles") else: profile_name = None calibrator = None if any(arg is not None for arg in [ self.calibration_cache, self.calibration_base_class ]) and not self.int8: G_LOGGER.warning( "Some int8 calibrator options were set, but int8 precision is not enabled. " "Calibration options will be ignored. Please set --int8 to enable calibration. " ) if self.int8 and self.data_loader_args is not None: # We cannot do calibration if there is no data loader. script.add_import(imports=["Calibrator"], frm="polygraphy.backend.trt") script.add_import(imports=["DataLoader"], frm="polygraphy.comparator") data_loader_name = self.data_loader_args.add_data_loader(script) if self.calibration_base_class: script.add_import(imports=["tensorrt as trt"]) calibrator = make_invocable( "Calibrator", data_loader=data_loader_name if data_loader_name else inline( safe("DataLoader()")), cache=self.calibration_cache, BaseClass=self.calibration_base_class, quantile=self.quantile, regression_cutoff=self.regression_cutoff, ) algo_selector = None if self.load_tactics is not None: script.add_import(imports=["TacticReplayer"], frm="polygraphy.backend.trt") algo_selector = make_invocable("TacticReplayer", replay=self.load_tactics) elif self.save_tactics is not None: script.add_import(imports=["TacticRecorder"], frm="polygraphy.backend.trt") algo_selector = make_invocable("TacticRecorder", record=self.save_tactics) if self.tactic_sources is not None: script.add_import(imports=["tensorrt as trt"]) if self.trt_config_script is not None: script.add_import(imports=["InvokeFromScript"], frm="polygraphy.backend.common") config_loader_str = make_invocable("InvokeFromScript", self.trt_config_script, name=self.trt_config_func_name) else: config_loader_str = make_invocable_if_nondefault( "CreateTrtConfig", max_workspace_size=self.workspace, tf32=self.tf32, fp16=self.fp16, int8=self.int8, strict_types=self.strict_types, restricted=self.restricted, profiles=profile_name, calibrator=calibrator, load_timing_cache=(self.timing_cache if self.timing_cache and os.path.exists(self.timing_cache) else None), algorithm_selector=algo_selector, sparse_weights=self.sparse_weights, tactic_sources=self.tactic_sources, ) if config_loader_str is not None: script.add_import(imports=["CreateConfig as CreateTrtConfig"], frm="polygraphy.backend.trt") if config_loader_str is not None: config_loader_name = script.add_loader(config_loader_str, "create_trt_config") else: config_loader_name = None return config_loader_name
def parse(self, args): trt_min_shapes = util.default(args_util.get(args, "trt_min_shapes"), []) trt_max_shapes = util.default(args_util.get(args, "trt_max_shapes"), []) trt_opt_shapes = util.default(args_util.get(args, "trt_opt_shapes"), []) default_shapes = TensorMetadata() if self.model_args is not None: assert hasattr(self.model_args, "input_shapes" ), "ModelArgs must be parsed before TrtConfigArgs!" default_shapes = self.model_args.input_shapes self.profile_dicts = parse_profile_shapes(default_shapes, trt_min_shapes, trt_opt_shapes, trt_max_shapes) workspace = args_util.get(args, "workspace") self.workspace = int(workspace) if workspace is not None else workspace self.tf32 = args_util.get(args, "tf32") self.fp16 = args_util.get(args, "fp16") self.int8 = args_util.get(args, "int8") self.strict_types = args_util.get(args, "strict_types") self.restricted = args_util.get(args, "restricted") self.calibration_cache = args_util.get(args, "calibration_cache") calib_base = args_util.get(args, "calibration_base_class") self.calibration_base_class = None if calib_base is not None: calib_base = safe(assert_identifier(calib_base)) self.calibration_base_class = inline( safe("trt.{:}", inline(calib_base))) self.quantile = args_util.get(args, "quantile") self.regression_cutoff = args_util.get(args, "regression_cutoff") self.sparse_weights = args_util.get(args, "sparse_weights") self.timing_cache = args_util.get(args, "timing_cache") tactic_replay = args_util.get(args, "tactic_replay") self.load_tactics = args_util.get(args, "load_tactics") self.save_tactics = args_util.get(args, "save_tactics") if tactic_replay is not None: mod.warn_deprecated("--tactic-replay", "--save-tactics or --load-tactics", remove_in="0.35.0") G_LOGGER.warning( "--tactic-replay is deprecated. Use either --save-tactics or --load-tactics instead." ) if os.path.exists( tactic_replay) and util.get_file_size(tactic_replay) > 0: self.load_tactics = tactic_replay else: self.save_tactics = tactic_replay tactic_sources = args_util.get(args, "tactic_sources") self.tactic_sources = None if tactic_sources is not None: self.tactic_sources = [] for source in tactic_sources: source = safe(assert_identifier(source.upper())) source_str = safe("trt.TacticSource.{:}", inline(source)) self.tactic_sources.append(inline(source_str)) self.trt_config_script = args_util.get(args, "trt_config_script") self.trt_config_func_name = args_util.get(args, "trt_config_func_name")
def add_to_script(self, script): script.add_import(imports=["TrtLegacyRunner"], frm="polygraphy.backend.trt_legacy") G_LOGGER.warning("Legacy TensorRT runner only supports implicit batch TensorFlow/UFF, ONNX, and Caffe models") load_engine = self.model_args.model_file if self.model_args.model_type == "engine" else None loader_name = None if self.model_args.model_type == "onnx": script.add_import(imports=["ParseNetworkFromOnnxLegacy"], frm="polygraphy.backend.trt_legacy") onnx_loader = self.onnx_loader_args.add_onnx_loader(script, disable_custom_outputs=True) loader_name = script.add_loader( make_invocable("ParseNetworkFromOnnxLegacy", onnx_loader), "parse_network_from_onnx_legacy" ) elif self.model_args.model_type == "caffe": script.add_import(imports=["LoadNetworkFromCaffe"], frm="polygraphy.backend.trt_legacy") loader_name = script.add_loader( make_invocable( "LoadNetworkFromCaffe", self.model_args.model_file, self.caffe_model, self.trt_outputs, self.batch_size, ), "parse_network_from_caffe", ) elif load_engine is None: script.add_import(imports=["LoadNetworkFromUff"], frm="polygraphy.backend.trt_legacy") if self.model_args.model_type == "uff": script.add_import(imports=["LoadUffFile"], frm="polygraphy.backend.trt_legacy") shapes = {name: shape for name, (_, shape) in self.model_args.input_shapes.items()} loader_name = script.add_loader( make_invocable( "LoadUffFile", self.model_args.model_file, util.default(shapes, {}), self.trt_outputs ), "load_uff_file", ) else: script.add_import(imports=["ConvertToUff"], frm="polygraphy.backend.trt_legacy") loader_name = script.add_loader( make_invocable( "ConvertToUff", self.tf_loader_args.add_to_script(script), save_uff=self.save_uff, preprocessor=self.preprocessor, ), "convert_to_uff", ) loader_name = script.add_loader( make_invocable("LoadNetworkFromUff", loader_name, uff_order=self.uff_order), "uff_network_loader" ) calibrator = None if ( self.trt_config_args.int8 and self.data_loader_args is not None ): # We cannot do calibration if there is no data loader. script.add_import(imports=["Calibrator"], frm="polygraphy.backend.trt") script.add_import(imports=["DataLoader"], frm="polygraphy.comparator") data_loader_name = self.data_loader_args.add_data_loader(script) if self.calibration_base_class: script.add_import(imports=["tensorrt as trt"]) calibrator = make_invocable( "Calibrator", data_loader=data_loader_name if data_loader_name else inline(safe("DataLoader()")), cache=self.calibration_cache, BaseClass=self.calibration_base_class, quantile=self.quantile, regression_cutoff=self.regression_cutoff, ) runner_str = make_invocable( "TrtLegacyRunner", network_loader=loader_name, max_workspace_size=self.trt_config_args.workspace, max_batch_size=self.batch_size, fp16=self.trt_config_args.fp16, tf32=self.trt_config_args.tf32, load_engine=load_engine, save_engine=self.trt_engine_save_args.path, layerwise=self.trt_outputs == constants.MARK_ALL, plugins=self.trt_engine_loader_args.plugins, int8=self.trt_config_args.int8, calibrator=calibrator, use_dla=self.use_dla, allow_gpu_fallback=self.allow_gpu_fallback, ) script.add_runner(runner_str)