def __call__(self, args):
run_results = misc.pickle_load(args.results)
def meta_from_iter_result(iter_result):
meta = TensorMetadata()
for name, arr in iter_result.items():
meta.add(name, dtype=arr.dtype, shape=arr.shape)
return meta
results_str = ""
results_str += "==== Run Results ({:} runners) ====\n\n".format(
len(run_results))
for runner_name, iters in run_results.items():
results_str += "---- Runner: {:} ({:} iterations) ----\n".format(
runner_name, len(iters))
for index, iter_result in enumerate(iters):
if args.show_values:
for name, arr in iter_result.items():
results_str += "{:} [dtype={:}, shape={:}]\n{:}\n\n".format(
name, arr.dtype, arr.shape,
misc.indent_block(str(arr)))
else:
iter_meta = meta_from_iter_result(iter_result)
if len(iters) > 1 and args.all:
results_str += misc.indent_block(
"Iteration: {:} | ".format(index))
results_str += "{:}\n".format(iter_meta)
if not args.all:
break
results_str += "\n"
results_str = misc.indent_block(results_str, level=0)
G_LOGGER.info(results_str)
def run(self, args):
import tensorrt as trt
if not self.makers[TrtLoaderArgs].calibration_cache:
G_LOGGER.warning(
"Not using a calibration cache. Using a calibration cache may significantly speed up the search process"
)
self.precision = {
"float32": trt.float32,
"float16": trt.float16
}[args.precision]
if self.precision == trt.float16 and not self.makers[
TrtLoaderArgs].fp16:
self.makers[TrtLoaderArgs].fp16 = True
if self.precision == trt.float16 and not self.makers[
TrtLoaderArgs].int8:
G_LOGGER.warning(
"Using float16 as the higher precision, but float16 is also the lowest precision available. Did you mean to set --int8 as well?"
)
if not any([
self.makers[TrtLoaderArgs].tf32,
self.makers[TrtLoaderArgs].fp16,
self.makers[TrtLoaderArgs].int8
]):
G_LOGGER.critical(
"Please enable at least one precision besides float32 (e.g. --int8, --fp16)"
)
if self.makers[ModelArgs].model_type == "engine":
G_LOGGER.critical(
"The precision tool cannot work with engines, as they cannot be modified. "
"Please provide a different format, such as an ONNX or TensorFlow model."
)
self.args = args
self.golden = OrderedDict()
self.golden.update(misc.pickle_load(args.golden))
self.builder, self.network, self.parser = func.invoke(
self.makers[TrtLoaderArgs].get_trt_network_loader())
with self.builder, self.network, self.parser:
indices = self.find()
if indices is not None:
G_LOGGER.info(
"To achieve acceptable accuracy, try running layers: {:} in {:} precision"
.format(indices, self.precision))
else:
G_LOGGER.critical(
"Could not find a configuration that resulted in acceptable accuracy"
)
def test_trt_exclude_outputs_with_layerwise(self):
with tempfile.NamedTemporaryFile() as outfile0:
run_polygraphy_run([
ONNX_MODELS["identity_identity"].path, "--trt",
"--trt-outputs", "mark", "all", "--trt-exclude-outputs",
"identity_out_2", "--save-results", outfile0.name
])
results = misc.pickle_load(outfile0.name)
[result] = list(results.values())[0]
assert len(result) == 1
assert "identity_out_0" in result
def run(self, args):
data = misc.pickle_load(args.path)
def meta_from_iter_result(iter_result):
meta = TensorMetadata()
for name, arr in iter_result.items():
meta.add(name, dtype=arr.dtype, shape=arr.shape)
return meta
def str_from_iters(iters):
out_str = ""
for index, iter_result in enumerate(iters):
if args.show_values:
for name, arr in iter_result.items():
out_str += "{:} [dtype={:}, shape={:}]\n{:}\n\n".format(
name, arr.dtype, arr.shape,
misc.indent_block(str(arr)))
else:
iter_meta = meta_from_iter_result(iter_result)
if len(iters) > 1 and args.all:
out_str += misc.indent_block(
"Iteration: {:} | ".format(index))
out_str += "{:}\n".format(iter_meta)
if not args.all:
break
return out_str
def display_results():
results_str = ""
results_str += "==== Run Results ({:} runners) ====\n\n".format(
len(data))
for runner_name, iters in data.items():
results_str += "---- Runner: {:} ({:} iterations) ----\n".format(
runner_name, len(iters))
results_str += str_from_iters(iters) + "\n"
results_str = misc.indent_block(results_str, level=0).strip()
G_LOGGER.info(results_str)
def display_inputs():
inputs_str = ""
inputs_str += "==== Input Data ({:} iterations) ====\n\n".format(
len(data))
inputs_str += str_from_iters(data) + "\n"
inputs_str = misc.indent_block(inputs_str, level=0).strip()
G_LOGGER.info(inputs_str)
if isinstance(data, RunResults):
display_results()
else:
display_inputs()
def test_onnx_rt_layerwise_outputs():
with tempfile.NamedTemporaryFile() as outfile0:
run_polygraphy_run([ONNX_MODELS["identity_identity"].path, "--onnxrt", "--onnx-outputs", "mark", "all", "--save-results", outfile0.name])
results = misc.pickle_load(outfile0.name)
[result] = list(results.values())[0]
assert len(result) == 2