def str_from_iters(iters):
out_str = ""
for index, iter_result in enumerate(iters):
iter_meta = meta_from_iter_result(iter_result)
indent = 1
if len(iters) > 1 and args.all:
out_str += util.indent_block(
"\n-- Iteration: {:}\n".format(index), indent - 1)
indent = 2
for name, arr in iter_result.items():
out_str += util.indent_block(
"\n{:} {:} | Stats: {:}".format(
name, iter_meta[name],
comp_util.str_output_stats(arr)),
indent - 1,
)
if args.histogram:
out_str += "\n{:}".format(
util.indent_block(comp_util.str_histogram(arr),
indent))
if args.show_values:
out_str += "\n{:}".format(
util.indent_block(str(arr), indent))
if indent == 2:
out_str += "\n"
if not args.all:
break
return out_str
def str_from_iters(iters):
out_str = ""
for index, iter_result in enumerate(iters):
if args.show_values:
for name, arr in iter_result.items():
out_str += "{:} [dtype={:}, shape={:}]\n{:}\n".format(
name, arr.dtype, arr.shape,
util.indent_block(str(arr)))
else:
iter_meta = meta_from_iter_result(iter_result)
if len(iters) > 1 and args.all:
out_str += util.indent_block(
"Iteration: {:} | ".format(index))
out_str += "{:}\n".format(iter_meta)
stat_str = "\n-- Statistics --"
for name, arr in iter_result.items():
stat_str += "\n{:} | Stats\n".format(name)
stat_str += util.indent_block(
comp_util.str_output_stats(arr)) + "\n"
if args.histogram:
stat_str += util.indent_block(
comp_util.str_histogram(arr)) + "\n"
out_str += stat_str
if not args.all:
break
return out_str
def str_from_engine(engine):
bindings_per_profile = get_bindings_per_profile(engine)
engine_str = "Name: {:} | {:}{:} Batch Engine ({:} layers)\n".format(
engine.name, "Refittable " if engine.refittable else "",
"Implicit" if hasattr(engine, "has_implicit_batch_dimension")
and engine.has_implicit_batch_dimension else "Explicit",
engine.num_layers)
engine_str += "\n"
# Show metadata for the first profile (i.e. the dynamic shapes)
input_metadata = get_input_metadata_from_engine(engine, 0,
bindings_per_profile)
engine_str += "---- {:} Engine Input(s) ----\n{:}\n\n".format(
len(input_metadata), input_metadata)
output_metadata = get_output_metadata_from_engine(engine, 0,
bindings_per_profile)
engine_str += "---- {:} Engine Output(s) ----\n{:}\n\n".format(
len(output_metadata), output_metadata)
engine_str += "---- Memory ----\nDevice Memory: {:} bytes\n\n".format(
engine.device_memory_size)
engine_str += "---- {:} Profile(s) ({:} Binding(s) Each) ----\n".format(
engine.num_optimization_profiles, bindings_per_profile)
for profile_index in range(engine.num_optimization_profiles):
engine_str += "- Profile: {:}\n".format(profile_index)
max_width = max([len(binding) for binding in engine]) + 8
for offset in range(bindings_per_profile):
binding = profile_index * bindings_per_profile + offset
name = "[Name: {:}]".format(engine.get_binding_name(binding))
engine_str += util.indent_block(
"Binding Index: {:} {:} {:<{max_width}}".format(
binding,
"(Input) "
if engine.binding_is_input(binding) else "(Output)",
name,
max_width=max_width))
if engine.binding_is_input(binding):
if engine.is_shape_binding(binding):
min_shape, opt_shape, max_shape = engine.get_profile_shape_input(
profile_index, binding)
else:
min_shape, opt_shape, max_shape = engine.get_profile_shape(
profile_index, binding)
engine_str += " | Shapes: min={:}, opt={:}, max={:}\n".format(
min_shape, opt_shape, max_shape)
else:
engine_str += " | Shape: {:}".format(
tuple(output_metadata[engine[offset]].shape))
engine_str += "\n"
return util.indent_block(engine_str, level=0)
def str_from_network(network, mode="full"):
"""
Converts a TensorRT network to a human-readable representation
Args:
network (trt.INetworkDefinition): The network.
mode (str): Controls what is displayed for each layer. Choices: ["none", "basic", "attrs", "full"]
Returns:
str
"""
LAYER_TYPE_CLASS_MAPPING = get_layer_class_mapping()
network_str = "Name: {:} | {:} Batch Network{:}\n".format(
network.name,
"Implicit"
if hasattr(network, "has_implicit_batch_dimension") and network.has_implicit_batch_dimension
else "Explicit",
" with Explicit Precision "
if hasattr(network, "has_explicit_precision") and network.has_explicit_precision
else "",
)
network_str += "\n"
input_metadata = get_network_input_metadata(network)
network_str += "---- {:} Network Input(s) ----\n{:}\n\n".format(len(input_metadata), input_metadata)
output_metadata = get_network_output_metadata(network)
network_str += "---- {:} Network Output(s) ----\n{:}\n\n".format(len(output_metadata), output_metadata)
network_str += "---- {:} Layer(s) ----\n".format(network.num_layers)
if mode != "none":
for index, layer in enumerate(network):
if layer.type in LAYER_TYPE_CLASS_MAPPING:
layer.__class__ = LAYER_TYPE_CLASS_MAPPING[layer.type]
network_str += str_from_layer(layer, index)
if mode in ["attrs", "full"]:
# Exclude special attributes, as well as any attributes of the base layer class (those can be displayed above).
attrs = get_layer_attribute_names(layer)
if attrs:
network_str += util.indent_block("---- Attributes ----") + "\n"
for attr in attrs:
with G_LOGGER.verbosity():
val = getattr(layer, attr)
if mode == "full" or not isinstance(val, np.ndarray):
attr_str = ""
if layer.name:
attr_str += "{:}.".format(layer.name)
network_str += util.indent_block("{:}{:} = {:}".format(attr_str, attr, val)) + "\n"
network_str += "\n"
return util.indent_block(network_str, level=0)
def display_inputs(input_data):
inputs_str = ""
inputs_str += "==== Data ({:} iterations) ====\n".format(
len(input_data))
inputs_str += str_from_iters(input_data) + "\n"
inputs_str = util.indent_block(inputs_str, level=0).strip()
G_LOGGER.info(inputs_str)
def display_results(results):
results_str = ""
results_str += "==== Run Results ({:} runners) ====\n\n".format(
len(results))
for runner_name, iters in results.items():
results_str += "---- {:35} ({:} iterations) ----\n".format(
runner_name, len(iters))
results_str += str_from_iters(iters) + "\n"
results_str = util.indent_block(results_str, level=0).strip()
G_LOGGER.info(results_str)
def execute_runner(runner, loader_cache):
with runner as active_runner:
input_metadata = active_runner.get_input_metadata()
G_LOGGER.info("{:35}\n---- Model Input(s) ----\n{:}".format(active_runner.name, input_metadata),
mode=LogMode.ONCE)
# DataLoaderCache will ensure that the feed_dict does not contain any extra entries
# based on the provided input_metadata.
loader_cache.set_input_metadata(input_metadata)
if warm_up:
G_LOGGER.start("{:35} | Running {:} warm-up run(s)".format(active_runner.name, warm_up))
try:
feed_dict = loader_cache[0]
except IndexError:
G_LOGGER.warning("{:} warm-up run(s) were requested, but data loader did not supply any data. "
"Skipping warm-up run(s)".format(warm_up))
else:
G_LOGGER.ultra_verbose("Warm-up Input Buffers:\n{:}".format(util.indent_block(feed_dict)))
# First do a few warm-up runs, and don't time them.
for _ in range(warm_up):
active_runner.infer(feed_dict=feed_dict)
G_LOGGER.finish("{:35} | Finished {:} warm-up run(s)".format(active_runner.name, warm_up))
# Then, actual iterations.
index = 0
iteration_results = []
total_runtime = 0
for index, feed_dict in enumerate(loader_cache):
G_LOGGER.extra_verbose(lambda: "{:35} | Feeding inputs:\n{:}".format(active_runner.name, util.indent_block(feed_dict)))
outputs = active_runner.infer(feed_dict=feed_dict)
runtime = active_runner.last_inference_time()
total_runtime += runtime
# Without a deep copy here, outputs will always reference the output of the last run
iteration_results.append(IterationResult(outputs=copy.deepcopy(outputs), runtime=runtime, runner_name=active_runner.name))
G_LOGGER.info(lambda: "{:35}\n---- Model Output(s) ----\n{:}".format(
active_runner.name, TensorMetadata().from_feed_dict(outputs)),
mode=LogMode.ONCE)
G_LOGGER.extra_verbose(lambda: "{:35} | Inference Time: {:.3f} ms | Received outputs:\n{:}".format(
active_runner.name, runtime * 1000.0, util.indent_block(outputs)))
total_runtime_ms = total_runtime * 1000.0
G_LOGGER.finish("{:35} | Completed {:} iteration(s) in {:.4g} ms | Average inference time: {:.4g} ms.".format(active_runner.name, index + 1, total_runtime_ms, total_runtime_ms / float(index + 1)))
return iteration_results
def str_from_graph(graph, mode):
graph_str = ""
input_metadata = get_input_metadata(graph)
output_metadata = get_output_metadata(graph)
graph_str += "---- {:} Graph Inputs ----\n{:}\n\n".format(
len(input_metadata), input_metadata)
graph_str += "---- {:} Graph Outputs ----\n{:}\n\n".format(
len(output_metadata), output_metadata)
graph_str += "---- {:} Nodes ----\n".format(len(graph.as_graph_def().node))
if mode == "basic":
G_LOGGER.warning(
"Displaying layer information is unsupported for TensorFlow graphs. "
"Please use --mode=full if you would like to see the raw nodes")
if mode == "full":
for node in graph.as_graph_def().node:
graph_str += str(node) + "\n"
graph_str += "\n"
return util.indent_block(graph_str, level=0)
def process_attr(attr_str: str):
processed = getattr(attr, ONNX_PYTHON_ATTR_MAPPING[attr_str])
if attr_str == "STRING":
processed = processed.decode()
elif attr_str == "TENSOR":
tensor_str = "Tensor: [dtype={:}, shape={:}]".format(
get_dtype(processed), get_shape(processed))
if mode == "full":
tensor_str += " | Values:\n" + util.indent_block(
str(get_values(processed)))
processed = tensor_str
elif attr_str == "GRAPH":
processed = "\n" + str_from_onnx_graph(
processed,
mode,
tensors,
indent_level=indent_level + 2)
elif attr_str == "FLOATS" or attr_str == "INTS":
# Proto hacky list to normal Python list
processed = [p for p in processed]
elif attr_str == "STRINGS":
processed = [p.decode() for p in processed]
return processed
def check_outputs_match(out0, out0_name, out1, out1_name, per_out_rtol, per_out_atol, per_out_err_stat):
VALID_CHECK_ERROR_STATS = ["max", "mean", "median", "elemwise"]
if per_out_err_stat not in VALID_CHECK_ERROR_STATS:
G_LOGGER.critical("Invalid choice for check_error_stat: {:}.\n"
"Note: Valid choices are: {:}".format(per_out_err_stat, VALID_CHECK_ERROR_STATS))
G_LOGGER.super_verbose("{:35} | Output: {:} (dtype={:}, shape={:}):\n{:}".format(
iter_result0.runner_name, out0_name, out0.dtype, out0.shape, util.indent_block(out0)))
G_LOGGER.super_verbose("{:35} | Output: {:} (dtype={:}, shape={:}):\n{:}".format(
iter_result1.runner_name, out1_name, out1.dtype, out1.shape, util.indent_block(out1)))
# Check difference vs. tolerances
if np.issubdtype(out0.dtype, np.bool_) and np.issubdtype(out1.dtype, np.bool_):
absdiff = np.logical_xor(out0, out1)
else:
absdiff = np.abs(out0 - out1)
absout1 = np.abs(out1)
with np.testing.suppress_warnings() as sup:
sup.filter(RuntimeWarning)
reldiff = absdiff / absout1
max_absdiff = comp_util.compute_max(absdiff)
mean_absdiff = comp_util.compute_mean(absdiff)
median_absdiff = comp_util.compute_median(absdiff)
max_reldiff = comp_util.compute_max(reldiff)
mean_reldiff = comp_util.compute_mean(reldiff)
median_reldiff = comp_util.compute_median(reldiff)
max_elemwiseabs = "Unknown"
max_elemwiserel = "Unknown"
if per_out_err_stat == "mean":
failed = mean_absdiff > per_out_atol and (np.isnan(mean_reldiff) or mean_reldiff > per_out_rtol)
elif per_out_err_stat == "median":
failed = median_absdiff > per_out_atol and (np.isnan(median_reldiff) or median_reldiff > per_out_rtol)
elif per_out_err_stat == "max":
failed = max_absdiff > per_out_atol and (np.isnan(max_reldiff) or max_reldiff > per_out_rtol)
else:
assert per_out_err_stat == "elemwise", "This branch should be unreachable unless per_out_err_stat is 'elemwise'"
mismatches = (absdiff > per_out_atol) & (reldiff > per_out_rtol)
failed = np.any(mismatches)
try:
# Special because we need to account for tolerances too.
max_elemwiseabs = comp_util.compute_max(absdiff[mismatches])
max_elemwiserel = comp_util.compute_max(reldiff[mismatches])
with G_LOGGER.indent():
G_LOGGER.super_verbose("Mismatched indices:\n{:}".format(np.argwhere(mismatches)))
G_LOGGER.extra_verbose("{:35} | Mismatched values:\n{:}".format(iter_result0.runner_name, out0[mismatches]))
G_LOGGER.extra_verbose("{:35} | Mismatched values:\n{:}".format(iter_result1.runner_name, out1[mismatches]))
except Exception as err:
G_LOGGER.warning("Failing to log mismatches.\nNote: Error was: {:}".format(err))
# Log information about the outputs
hist_bin_range = (min(comp_util.compute_min(out0), comp_util.compute_min(out1)),
max(comp_util.compute_max(out0), comp_util.compute_max(out1)))
comp_util.log_output_stats(out0, failed, iter_result0.runner_name + ": " + out0_name, hist_range=hist_bin_range)
comp_util.log_output_stats(out1, failed, iter_result1.runner_name + ": " + out1_name, hist_range=hist_bin_range)
G_LOGGER.info("Error Metrics: {:}".format(out0_name))
with G_LOGGER.indent():
def req_tol(mean_diff, median_diff, max_diff, elemwise_diff):
return {
"mean": mean_diff,
"median": median_diff,
"max": max_diff,
"elemwise": elemwise_diff,
}[per_out_err_stat]
G_LOGGER.info("Minimum Required Tolerance: {:} error | [abs={:.5g}] OR [rel={:.5g}]".format(
per_out_err_stat,
req_tol(mean_absdiff, median_absdiff, max_absdiff, max_elemwiseabs),
req_tol(mean_reldiff, median_reldiff, max_reldiff, max_elemwiserel)))
comp_util.log_output_stats(absdiff, failed, "Absolute Difference")
comp_util.log_output_stats(reldiff, failed, "Relative Difference")
# Finally show summary.
if failed:
G_LOGGER.error("FAILED | Difference exceeds tolerance (rel={:}, abs={:})".format(per_out_rtol, per_out_atol))
else:
G_LOGGER.finish("PASSED | Difference is within tolerance (rel={:}, abs={:})".format(per_out_rtol, per_out_atol))
G_LOGGER.extra_verbose("Finished comparing: '{:}' (dtype={:}, shape={:}) [{:}] and '{:}' (dtype={:}, shape={:}) [{:}]"
.format(out0_name, out0.dtype, out0.shape, iter_result0.runner_name, out1_name, out1.dtype, out1.shape, iter_result1.runner_name))
return OutputCompareResult(not failed, max_absdiff, max_reldiff, mean_absdiff, mean_reldiff, median_absdiff, median_reldiff)
def str_from_onnx_graph(graph, mode, tensors, indent_level=0):
input_metadata = get_input_metadata(graph)
output_metadata = get_output_metadata(graph)
initializer_metadata = get_tensor_metadata(graph.initializer)
# Subgraph inputs should remain separate from each other, hence copy the tensors map
tensors = copy.copy(tensors)
tensors.update(get_tensor_metadata(graph.value_info))
tensors.update(initializer_metadata)
tensors.update(input_metadata)
tensors.update(output_metadata)
graph_type = "Graph" if indent_level == 0 else "Subgraph"
onnx_str = ""
onnx_str += "---- {:} {:} Input(s) ----\n{:}\n\n".format(
len(input_metadata), graph_type, input_metadata)
onnx_str += "---- {:} {:} Output(s) ----\n{:}\n\n".format(
len(output_metadata), graph_type, output_metadata)
onnx_str += "---- {:} Initializer(s) ----\n".format(
len(initializer_metadata))
if mode == "full":
for init in graph.initializer:
onnx_str += "Initializer | {:} [dtype={:}, shape={:}] | Values:\n{:}\n\n".format(
init.name, get_dtype(init), get_shape(init),
util.indent_block(str(get_values(init))))
if not graph.initializer:
onnx_str += "{}\n\n"
elif mode != "none":
onnx_str += str(initializer_metadata)
onnx_str += "\n\n"
else:
onnx_str += "\n"
def metadata_from_names(names):
metadata = TensorMetadata()
for name in names:
dtype, shape = tensors.get(name, (None, None))
if name in initializer_metadata:
name = "Initializer | {:}".format(name)
metadata.add(name=name, dtype=dtype, shape=shape)
return metadata
# Maps values from the AttributeType enum to their string representations, e.g., {1: "FLOAT"}
ATTR_TYPE_MAPPING = dict(
zip(onnx.AttributeProto.AttributeType.values(),
onnx.AttributeProto.AttributeType.keys()))
# Maps an ONNX attribute to the corresponding Python property
ONNX_PYTHON_ATTR_MAPPING = {
"FLOAT": "f",
"INT": "i",
"STRING": "s",
"TENSOR": "t",
"GRAPH": "g",
"FLOATS": "floats",
"INTS": "ints",
"STRINGS": "strings",
}
def attrs_to_dict(attrs):
attr_dict = OrderedDict()
for attr in attrs:
def process_attr(attr_str: str):
processed = getattr(attr, ONNX_PYTHON_ATTR_MAPPING[attr_str])
if attr_str == "STRING":
processed = processed.decode()
elif attr_str == "TENSOR":
tensor_str = "Tensor: [dtype={:}, shape={:}]".format(
get_dtype(processed), get_shape(processed))
if mode == "full":
tensor_str += " | Values:\n" + util.indent_block(
str(get_values(processed)))
processed = tensor_str
elif attr_str == "GRAPH":
processed = "\n" + str_from_onnx_graph(
processed,
mode,
tensors,
indent_level=indent_level + 2)
elif attr_str == "FLOATS" or attr_str == "INTS":
# Proto hacky list to normal Python list
processed = [p for p in processed]
elif attr_str == "STRINGS":
processed = [p.decode() for p in processed]
return processed
if attr.type in ATTR_TYPE_MAPPING:
attr_str = ATTR_TYPE_MAPPING[attr.type]
if attr_str in ONNX_PYTHON_ATTR_MAPPING:
attr_dict[attr.name] = process_attr(attr_str)
else:
G_LOGGER.warning(
"Attribute of type {:} is currently unsupported. Skipping attribute."
.format(attr_str))
else:
G_LOGGER.warning(
"Attribute type: {:} was not recognized. Was the graph generated with a newer IR "
"version than the installed `onnx` package? Skipping attribute."
.format(attr.type))
return attr_dict
onnx_str += "---- {:} Node(s) ----\n".format(len(graph.node))
if mode != "none":
for index, node in enumerate(graph.node):
input_info = metadata_from_names(node.input)
output_info = metadata_from_names(node.output)
onnx_str += util.str_from_layer("Node", index, node.name,
node.op_type, input_info,
output_info)
if mode in ["attrs", "full"]:
attrs = attrs_to_dict(node.attribute)
if attrs:
onnx_str += util.indent_block(
"---- Attributes ----") + "\n"
for key, val in attrs.items():
attr_str = ""
if node.name:
attr_str += "{:}.".format(node.name)
onnx_str += util.indent_block("{:}{:} = {:}".format(
attr_str, key, val)) + "\n"
onnx_str += "\n"
return util.indent_block(onnx_str, indent_level)
def str_from_network(network, mode="full"):
"""
Converts a TensorRT network to a human-readable representation
Args:
network (trt.INetworkDefinition): The network.
mode (str): Controls what is displayed for each layer. Choices: ["none", "basic", "attrs", "full"]
Returns:
str
"""
LAYER_TYPE_CLASS_MAPPING = get_layer_class_mapping()
def is_special_attribute(attr):
return attr.startswith("__") and attr.endswith("__")
def is_valid_attribute(attr, layer):
if type(layer) == trt.IPoolingLayer or type(
layer) == trt.IConvolutionLayer or type(
layer) == trt.IDeconvolutionLayer:
if len(layer.get_input(0).shape) > 4:
# 3D pooling uses padding_nd
return attr not in ["padding", "stride", "window_size"]
if type(layer) == trt.IResizeLayer:
if layer.num_inputs > 1:
return attr not in ["scales"]
if type(layer) == trt.ISliceLayer:
if layer.num_inputs > 1:
return attr not in ["shape", "start", "stride"]
return True
network_str = "Name: {:} | {:} Batch Network{:}\n".format(
network.name,
"Implicit" if hasattr(network, "has_implicit_batch_dimension")
and network.has_implicit_batch_dimension else "Explicit",
" with Explicit Precision "
if hasattr(network, "has_explicit_precision")
and network.has_explicit_precision else "")
network_str += "\n"
input_metadata = get_input_metadata(network)
network_str += "---- {:} Network Input(s) ----\n{:}\n\n".format(
len(input_metadata), input_metadata)
output_metadata = get_output_metadata(network)
network_str += "---- {:} Network Output(s) ----\n{:}\n\n".format(
len(output_metadata), output_metadata)
network_str += "---- {:} Layer(s) ----\n".format(network.num_layers)
if mode != "none":
for index, layer in enumerate(network):
if layer.type in LAYER_TYPE_CLASS_MAPPING:
layer.__class__ = LAYER_TYPE_CLASS_MAPPING[layer.type]
network_str += str_from_layer(layer, index)
if mode in ["attrs", "full"]:
# Exclude special attributes, as well as any attributes of the base layer class (those can be displayed above).
attrs = [
attr for attr in dir(layer)
if not is_special_attribute(attr) and not hasattr(
trt.ILayer, attr) and is_valid_attribute(attr, layer)
]
if attrs:
network_str += util.indent_block(
"---- Attributes ----") + "\n"
for attr in attrs:
with G_LOGGER.verbosity():
val = getattr(layer, attr)
if mode == "full" or not isinstance(val, np.ndarray):
attr_str = ""
if layer.name:
attr_str += "{:}.".format(layer.name)
network_str += util.indent_block("{:}{:} = {:}".format(
attr_str, attr, val)) + "\n"
network_str += "\n"
return util.indent_block(network_str, level=0)
