def __call__(self, args):
run_results = misc.pickle_load(args.results)
def meta_from_iter_result(iter_result):
meta = TensorMetadata()
for name, arr in iter_result.items():
meta.add(name, dtype=arr.dtype, shape=arr.shape)
return meta
results_str = ""
results_str += "==== Run Results ({:} runners) ====\n\n".format(
len(run_results))
for runner_name, iters in run_results.items():
results_str += "---- Runner: {:} ({:} iterations) ----\n".format(
runner_name, len(iters))
for index, iter_result in enumerate(iters):
if args.show_values:
for name, arr in iter_result.items():
results_str += "{:} [dtype={:}, shape={:}]\n{:}\n\n".format(
name, arr.dtype, arr.shape,
misc.indent_block(str(arr)))
else:
iter_meta = meta_from_iter_result(iter_result)
if len(iters) > 1 and args.all:
results_str += misc.indent_block(
"Iteration: {:} | ".format(index))
results_str += "{:}\n".format(iter_meta)
if not args.all:
break
results_str += "\n"
results_str = misc.indent_block(results_str, level=0)
G_LOGGER.info(results_str)
def display_inputs():
inputs_str = ""
inputs_str += "==== Input Data ({:} iterations) ====\n\n".format(
len(data))
inputs_str += str_from_iters(data) + "\n"
inputs_str = misc.indent_block(inputs_str, level=0).strip()
G_LOGGER.info(inputs_str)
def str_from_iters(iters):
out_str = ""
for index, iter_result in enumerate(iters):
if args.show_values:
for name, arr in iter_result.items():
out_str += "{:} [dtype={:}, shape={:}]\n{:}\n\n".format(
name, arr.dtype, arr.shape,
misc.indent_block(str(arr)))
else:
iter_meta = meta_from_iter_result(iter_result)
if len(iters) > 1 and args.all:
out_str += misc.indent_block(
"Iteration: {:} | ".format(index))
out_str += "{:}\n".format(iter_meta)
if not args.all:
break
return out_str
def display_results():
results_str = ""
results_str += "==== Run Results ({:} runners) ====\n\n".format(
len(data))
for runner_name, iters in data.items():
results_str += "---- Runner: {:} ({:} iterations) ----\n".format(
runner_name, len(iters))
results_str += str_from_iters(iters) + "\n"
results_str = misc.indent_block(results_str, level=0).strip()
G_LOGGER.info(results_str)
def str_from_engine(engine):
bindings_per_profile = get_bindings_per_profile(engine)
engine_str = "Name: {:} | {:}{:} Batch Engine ({:} layers)\n".format(engine.name,
"Refittable " if engine.refittable else "",
"Implicit" if hasattr(engine, "has_implicit_batch_dimension") and engine.has_implicit_batch_dimension else "Explicit",
engine.num_layers)
engine_str += "\n"
# Show metadata for the first profile (i.e. the dynamic shapes)
input_metadata = get_input_metadata_from_engine(engine, 0, bindings_per_profile)
engine_str += "---- {:} Engine Inputs ----\n{:}\n\n".format(len(input_metadata), input_metadata)
output_metadata = get_output_metadata_from_engine(engine, 0, bindings_per_profile)
engine_str += "---- {:} Engine Outputs ----\n{:}\n\n".format(len(output_metadata), output_metadata)
engine_str += "---- Memory ----\nDevice Memory: {:} bytes\n\n".format(engine.device_memory_size)
engine_str += "---- {:} Profiles ({:} Bindings Each) ----\n".format(engine.num_optimization_profiles, bindings_per_profile)
for profile_index in range(engine.num_optimization_profiles):
engine_str += "- Profile: {:}\n".format(profile_index)
max_width = max([len(binding) for binding in engine]) + 8
for offset in range(bindings_per_profile):
binding = profile_index * bindings_per_profile + offset
name = "[Name: {:}]".format(engine.get_binding_name(binding))
engine_str += misc.indent_block("Binding Index: {:} {:} {:<{max_width}}".format(
binding, "(Input) " if engine.binding_is_input(binding) else "(Output)", name, max_width=max_width))
if engine.binding_is_input(binding):
if engine.is_shape_binding(binding):
min_shape, opt_shape, max_shape = engine.get_profile_shape_input(profile_index, binding)
else:
min_shape, opt_shape, max_shape = engine.get_profile_shape(profile_index, binding)
engine_str += " | Shapes: min={:}, opt={:}, max={:}\n".format(min_shape, opt_shape, max_shape)
else:
engine_str += " | Shape: {:}".format(tuple(output_metadata[engine[offset]][1]))
engine_str += "\n"
return misc.indent_block(engine_str, level=0)
def str_from_graph(graph, mode):
graph_str = ""
input_metadata = get_input_metadata(graph)
output_metadata = get_output_metadata(graph)
graph_str += "---- {:} Graph Inputs ----\n{:}\n\n".format(len(input_metadata), input_metadata)
graph_str += "---- {:} Graph Outputs ----\n{:}\n\n".format(len(output_metadata), output_metadata)
graph_str += "---- {:} Nodes ----\n".format(len(graph.as_graph_def().node))
if mode == "basic":
G_LOGGER.warning("Displaying layer information is unsupported for TensorFlow graphs. "
"Please use --mode=full if you would like to see the raw nodes")
if mode == "full":
for node in graph.as_graph_def().node:
graph_str += str(node) + "\n"
graph_str += "\n"
return misc.indent_block(graph_str, level=0)
def process_attr(attr_str: str):
processed = getattr(attr, ONNX_PYTHON_ATTR_MAPPING[attr_str])
if attr_str == "STRING":
processed = processed.decode()
elif attr_str == "TENSOR":
tensor_str = "Tensor: [dtype={:}, shape={:}]".format(get_dtype(processed), get_shape(processed))
if mode == "full":
tensor_str += " | Values:\n" + misc.indent_block(str(get_values(processed)))
processed = tensor_str
elif attr_str == "GRAPH":
processed = "\n" + str_from_onnx_graph(processed, mode, tensors, indent_level=indent_level + 2)
elif attr_str == "FLOATS" or attr_str == "INTS":
# Proto hacky list to normal Python list
processed = [p for p in processed]
elif attr_str == "STRINGS":
processed = [p.decode() for p in processed]
return processed
def execute_runner(runner, loader_cache):
with runner as active_runner:
input_metadata = active_runner.get_input_metadata()
G_LOGGER.info("Runner: {:40} | Input Metadata: {:}".format(
active_runner.name, input_metadata),
mode=LogMode.ONCE)
# DataLoaderCache will ensure that the feed_dict does not contain any extra entries
# based on the provided input_metadata.
loader_cache.set_input_metadata(input_metadata)
if warm_up:
G_LOGGER.start(
"Runner: {:40} | Running {:} warm-up runs".format(
active_runner.name, warm_up))
try:
feed_dict = loader_cache[0]
except IndexError:
G_LOGGER.warning(
"{:} warm-up runs were requested, but data loader did not supply any data. "
"Skipping warm-up runs".format(warm_up))
else:
G_LOGGER.ultra_verbose(
"Warm-up Input Buffers:\n{:}".format(
misc.indent_block(feed_dict)))
# First do a few warm-up runs, and don't time them.
for i in range(warm_up):
active_runner.infer(feed_dict=feed_dict)
# Then, actual iterations.
index = 0
iteration_results = []
output_metadata = TensorMetadata()
for index, feed_dict in enumerate(loader_cache):
G_LOGGER.extra_verbose(
lambda: "Runner: {:40} | Feeding inputs:\n{:}".format(
active_runner.name, misc.indent_block(feed_dict)))
outputs = active_runner.infer(feed_dict=feed_dict)
runtime = active_runner.last_inference_time()
# Without a deep copy here, outputs will always reference the output of the last run
iteration_results.append(
IterationResult(outputs=copy.deepcopy(outputs),
runtime=runtime,
runner_name=active_runner.name))
if index == 0:
for name, out in outputs.items():
output_metadata.add(name, out.dtype, out.shape)
G_LOGGER.info(
"Runner: {:40} | Output Metadata: {:}".format(
active_runner.name, output_metadata),
mode=LogMode.ONCE)
G_LOGGER.extra_verbose(
lambda:
"Runner: {:40} | Inference Time: {:.3f} ms | Received outputs:\n{:}"
.format(active_runner.name, runtime * 1000.0,
misc.indent_block(outputs)))
G_LOGGER.finish(
"Runner: {:40} | Completed {:} iterations.".format(
active_runner.name, index + 1))
return iteration_results
def check_outputs_match(out0, out0_name, out1, out1_name, per_out_rtol, per_out_atol):
def compute_max(buffer):
if misc.is_empty_shape(buffer.shape):
return 0
return np.amax(buffer)
# Returns index of max value
def compute_argmax(buffer):
if misc.is_empty_shape(buffer.shape):
return 0
return np.unravel_index(np.argmax(buffer), buffer.shape)
def compute_min(buffer):
if misc.is_empty_shape(buffer.shape):
return 0
return np.amin(buffer)
# Returns index of min value
def compute_argmin(buffer):
if misc.is_empty_shape(buffer.shape):
return 0
return np.unravel_index(np.argmin(buffer), buffer.shape)
def compute_mean(buffer):
if misc.is_empty_shape(buffer.shape):
return 0
return np.mean(buffer)
def compute_required():
# The purpose of this function is to determine the minimum tolerances such that
# the outputs would be considered a match.
# The NumPy formula for np.isclose is absolute(out0 - out1) <= (per_out_atol + per_out_rtol * absolute(out1))
# So, for both absolute/relative tolerance, given either one,
# we can compute the required value for the other:
# per_out_atol = absolute(out0 - out1)
# atol_if_rtol = absolute(out0 - out1) - per_out_rtol * absolute(out1)
# per_out_rtol = (absolute(out0 - out1) - per_out_atol) / absolute(out1)
if np.issubdtype(out0.dtype, np.bool_) and np.issubdtype(out1.dtype, np.bool_):
absdiff = np.logical_xor(out0, out1)
else:
absdiff = np.abs(out0 - out1)
absout1 = np.abs(out1)
max_absdiff = max(compute_max(absdiff), 0.0)
required_atol_if_rtol = max(compute_max(absdiff - per_out_rtol * absout1), 0.0)
# Suppress divide by 0 warnings
with np.testing.suppress_warnings() as sup:
sup.filter(RuntimeWarning)
reldiff = np.maximum(absdiff - per_out_atol, 0.0) / absout1
max_reldiff = max(compute_max(reldiff), 0.0)
return max_absdiff, required_atol_if_rtol, max_reldiff, compute_mean(absdiff), compute_mean(reldiff)
def log_mismatches(mismatches):
try:
with G_LOGGER.indent():
G_LOGGER.super_verbose("Mismatched indices:\n{:}".format(np.argwhere(mismatches)))
G_LOGGER.extra_verbose("Runner: {:40} | Mismatched values:\n{:}".format(iter_result0.runner_name, out0[mismatches]))
G_LOGGER.extra_verbose("Runner: {:40} | Mismatched values:\n{:}".format(iter_result1.runner_name, out1[mismatches]))
except:
G_LOGGER.warning("Failing to log mismatches - this may be because the outputs are of different shapes")
try:
mismatches = np.logical_not(np.isclose(output0, output1, rtol=per_out_rtol, atol=per_out_atol))
except Exception as err:
G_LOGGER.warning("Failed to compare outputs with:\n{:}\nSkipping".format(err))
return False
G_LOGGER.super_verbose("Runner: {:40} | Output: {:} (dtype={:}, shape={:}):\n{:}".format(
iter_result0.runner_name, out0_name, out0.dtype, out0.shape, misc.indent_block(out0)))
G_LOGGER.super_verbose("Runner: {:40} | Output: {:} (dtype={:}, shape={:}):\n{:}".format(
iter_result1.runner_name, out1_name, out1.dtype, out1.shape, misc.indent_block(out1)))
failed = np.any(mismatches)
try:
max_absdiff, required_atol_if_rtol, max_reldiff, mean_absdiff, mean_reldiff = compute_required()
except Exception as err:
max_absdiff, required_atol_if_rtol, max_reldiff, mean_absdiff, mean_reldiff = None, None, None, None, None
G_LOGGER.warning("Could not determine required tolerances due to an error:\n{:}".format(err))
log_msg = ""
else:
log_msg = "Required tolerances: [atol={:.5g}] OR [rtol={:.5g}, atol={:.5g}] OR [rtol={:.5g}, atol={:.5g}] | Mean Error: Absolute={:.5g}, Relative={:.5g}\n".format(
max_absdiff, per_out_rtol, required_atol_if_rtol, max_reldiff, per_out_atol, mean_absdiff, mean_reldiff)
log_msg += "Runner: {:40} | Stats: mean={:.5g}, min={:.5g} at {:}, max={:.5g} at {:}\n".format(
iter_result0.runner_name, compute_mean(out0), compute_min(out0), compute_argmin(out0), compute_max(out0), compute_argmax(out0))
log_msg += "Runner: {:40} | Stats: mean={:.5g}, min={:.5g} at {:}, max={:.5g} at {:}\n".format(
iter_result1.runner_name, compute_mean(out1), compute_min(out1), compute_argmin(out1), compute_max(out1), compute_argmax(out1))
G_LOGGER.info(log_msg)
if failed:
log_mismatches(mismatches)
G_LOGGER.error("FAILED | Difference exceeds tolerance (rtol={:}, atol={:})".format(per_out_rtol, per_out_atol))
else:
G_LOGGER.finish("PASSED | Difference is within tolerance (rtol={:}, atol={:})".format(per_out_rtol, per_out_atol))
G_LOGGER.extra_verbose("Finished comparing: '{:}' (dtype={:}, shape={:}) [{:}] and '{:}' (dtype={:}, shape={:}) [{:}]"
.format(out0_name, out0.dtype, out0.shape, iter_result0.runner_name, out1_name, out1.dtype, out1.shape, iter_result1.runner_name))
return OutputCompareResult(not failed, max_absdiff, max_reldiff)
def str_from_onnx_graph(graph, mode, tensors, indent_level=0):
import onnx
input_metadata = get_input_metadata(graph)
output_metadata = get_output_metadata(graph)
initializer_metadata = get_tensor_metadata(graph.initializer)
# Subgraph inputs should remain separate from each other, hence copy the tensors map
tensors = copy.copy(tensors)
tensors.update(get_tensor_metadata(graph.value_info))
tensors.update(initializer_metadata)
tensors.update(input_metadata)
tensors.update(output_metadata)
graph_type = "Graph" if indent_level == 0 else "Subgraph"
onnx_str = ""
onnx_str += "---- {:} {:} Inputs ----\n{:}\n\n".format(len(input_metadata), graph_type, input_metadata)
onnx_str += "---- {:} {:} Outputs ----\n{:}\n\n".format(len(output_metadata), graph_type, output_metadata)
onnx_str += "---- {:} Initializers ----\n".format(len(initializer_metadata))
if mode == "full":
for init in graph.initializer:
onnx_str += "Initializer | {:} [dtype={:}, shape={:}] | Values:\n{:}\n\n".format(
init.name, get_dtype(init), get_shape(init), misc.indent_block(str(get_values(init))))
if not graph.initializer:
onnx_str += "\n"
elif mode != "none":
onnx_str += str(initializer_metadata)
onnx_str += "\n\n"
else:
onnx_str += "(Use --mode to display)"
onnx_str += "\n\n"
def metadata_from_names(names):
metadata = TensorMetadata()
for name in names:
dtype = None
shape = None
if name in tensors:
dtype, shape = tensors[name]
if name in initializer_metadata:
name = "Initializer | {:}".format(name)
metadata.add(name=name, dtype=dtype, shape=shape)
return metadata
# Maps values from the AttributeType enum to their string representations, e.g., {1: "FLOAT"}
ATTR_TYPE_MAPPING = dict(zip(onnx.AttributeProto.AttributeType.values(), onnx.AttributeProto.AttributeType.keys()))
# Maps an ONNX attribute to the corresponding Python property
ONNX_PYTHON_ATTR_MAPPING = {
"FLOAT": "f",
"INT": "i",
"STRING": "s",
"TENSOR": "t",
"GRAPH": "g",
"FLOATS": "floats",
"INTS": "ints",
"STRINGS": "strings",
}
def attrs_to_dict(attrs):
attr_dict = OrderedDict()
for attr in attrs:
def process_attr(attr_str: str):
processed = getattr(attr, ONNX_PYTHON_ATTR_MAPPING[attr_str])
if attr_str == "STRING":
processed = processed.decode()
elif attr_str == "TENSOR":
tensor_str = "Tensor: [dtype={:}, shape={:}]".format(get_dtype(processed), get_shape(processed))
if mode == "full":
tensor_str += " | Values:\n" + misc.indent_block(str(get_values(processed)))
processed = tensor_str
elif attr_str == "GRAPH":
processed = "\n" + str_from_onnx_graph(processed, mode, tensors, indent_level=indent_level + 2)
elif attr_str == "FLOATS" or attr_str == "INTS":
# Proto hacky list to normal Python list
processed = [p for p in processed]
elif attr_str == "STRINGS":
processed = [p.decode() for p in processed]
return processed
if attr.type in ATTR_TYPE_MAPPING:
attr_str = ATTR_TYPE_MAPPING[attr.type]
if attr_str in ONNX_PYTHON_ATTR_MAPPING:
attr_dict[attr.name] = process_attr(attr_str)
else:
G_LOGGER.warning("Attribute of type {:} is currently unsupported. Skipping attribute.".format(attr_str))
else:
G_LOGGER.warning("Attribute type: {:} was not recognized. Was the graph generated with a newer IR "
"version than the installed `onnx` package? Skipping attribute.".format(attr.type))
return attr_dict
onnx_str += "---- {:} Nodes ----\n".format(len(graph.node))
if mode != "none":
for index, node in enumerate(graph.node):
input_info = metadata_from_names(node.input)
output_info = metadata_from_names(node.output)
onnx_str += misc.str_from_layer("Node", index, node.name, node.op_type, input_info, output_info)
if mode in ["attrs", "full"]:
attrs = attrs_to_dict(node.attribute)
if attrs:
onnx_str += misc.indent_block("---- Attributes ----") + "\n"
for key, val in attrs.items():
if node.name:
onnx_str += "{:}.".format(node.name)
onnx_str += misc.indent_block("{:} = {:}".format(key, val)) + "\n"
onnx_str += "\n"
else:
onnx_str += "(Use --mode to display)"
return misc.indent_block(onnx_str, indent_level)
def str_from_network(network, mode="full"):
"""
Converts a TensorRT network to a human-readable representation
Args:
network (trt.INetworkDefinition): The network.
mode (str): Controls what is displayed for each layer. Choices: ["none", "basic", "attrs", "full"]
Returns:
str
"""
import numpy as np
try:
LAYER_TYPE_CLASS_MAPPING = get_layer_class_mapping()
except AttributeError:
LAYER_TYPE_CLASS_MAPPING = {}
def is_special_attribute(attr):
return attr.startswith("__") and attr.endswith("__")
def is_valid_attribute(attr, layer):
if type(layer) == trt.IPoolingLayer or type(layer) == trt.IConvolutionLayer or type(layer) == trt.IDeconvolutionLayer:
if len(layer.get_input(0).shape) > 4:
# 3D pooling uses padding_nd
return attr not in ["padding", "stride", "window_size"]
if type(layer) == trt.IResizeLayer:
if layer.num_inputs > 1:
return attr not in ["scales"]
if type(layer) == trt.ISliceLayer:
if layer.num_inputs > 1:
return attr not in ["shape", "start", "stride"]
return True
def get_layer_input_metadata(layer):
meta = TensorMetadata()
for i in range(layer.num_inputs):
inp = layer.get_input(i)
if inp:
meta.add(inp.name, trt.nptype(inp.dtype), inp.shape)
return meta
def get_layer_output_metadata(layer):
meta = TensorMetadata()
for i in range(layer.num_outputs):
outp = layer.get_output(i)
if outp:
meta.add(outp.name, trt.nptype(outp.dtype), outp.shape)
return meta
network_str = "Name: {:} | {:} Batch Network{:}\n".format(network.name,
"Implicit" if hasattr(network, "has_implicit_batch_dimension") and network.has_implicit_batch_dimension else "Explicit",
" with Explicit Precision " if hasattr(network, "has_explicit_precision") and network.has_explicit_precision else "")
network_str += "\n"
input_metadata = get_input_metadata(network)
network_str += "---- {:} Network Inputs ----\n{:}\n\n".format(len(input_metadata), input_metadata)
output_metadata = get_output_metadata(network)
network_str += "---- {:} Network Outputs ----\n{:}\n\n".format(len(output_metadata), output_metadata)
network_str += "---- {:} Layers ----\n".format(network.num_layers)
if mode != "none":
for index, layer in enumerate(network):
if layer.type in LAYER_TYPE_CLASS_MAPPING:
layer.__class__ = LAYER_TYPE_CLASS_MAPPING[layer.type]
input_info = get_layer_input_metadata(layer)
output_info = get_layer_output_metadata(layer)
network_str += misc.str_from_layer("Layer", index, layer.name, layer.type, input_info, output_info)
if mode in ["attrs", "full"]:
# Exclude special attributes, as well as any attributes of the base layer class (those can be displayed above).
attrs = [attr for attr in dir(layer) if not is_special_attribute(attr) and not hasattr(trt.ILayer, attr) and is_valid_attribute(attr, layer)]
if attrs:
network_str += misc.indent_block("---- Attributes ----") + "\n"
for attr in attrs:
val = getattr(layer, attr)
if mode == "full" or not isinstance(val, np.ndarray):
attr_str = ""
if layer.name:
attr_str += "{:}.".format(layer.name)
network_str += misc.indent_block("{:}{:} = {:}".format(attr_str, attr, val)) + "\n"
network_str += "\n"
else:
network_str += "(Use --mode to display)"
return misc.indent_block(network_str, level=0)