def str_from_iters(iters): out_str = "" for index, iter_result in enumerate(iters): iter_meta = meta_from_iter_result(iter_result) indent = 1 if len(iters) > 1 and args.all: out_str += util.indent_block( "\n-- Iteration: {:}\n".format(index), indent - 1) indent = 2 for name, arr in iter_result.items(): out_str += util.indent_block( "\n{:} {:} | Stats: {:}".format( name, iter_meta[name], comp_util.str_output_stats(arr)), indent - 1, ) if args.histogram: out_str += "\n{:}".format( util.indent_block(comp_util.str_histogram(arr), indent)) if args.show_values: out_str += "\n{:}".format( util.indent_block(str(arr), indent)) if indent == 2: out_str += "\n" if not args.all: break return out_str
def str_from_iters(iters): out_str = "" for index, iter_result in enumerate(iters): if args.show_values: for name, arr in iter_result.items(): out_str += "{:} [dtype={:}, shape={:}]\n{:}\n".format( name, arr.dtype, arr.shape, util.indent_block(str(arr))) else: iter_meta = meta_from_iter_result(iter_result) if len(iters) > 1 and args.all: out_str += util.indent_block( "Iteration: {:} | ".format(index)) out_str += "{:}\n".format(iter_meta) stat_str = "\n-- Statistics --" for name, arr in iter_result.items(): stat_str += "\n{:} | Stats\n".format(name) stat_str += util.indent_block( comp_util.str_output_stats(arr)) + "\n" if args.histogram: stat_str += util.indent_block( comp_util.str_histogram(arr)) + "\n" out_str += stat_str if not args.all: break return out_str
def str_from_engine(engine): bindings_per_profile = get_bindings_per_profile(engine) engine_str = "Name: {:} | {:}{:} Batch Engine ({:} layers)\n".format( engine.name, "Refittable " if engine.refittable else "", "Implicit" if hasattr(engine, "has_implicit_batch_dimension") and engine.has_implicit_batch_dimension else "Explicit", engine.num_layers) engine_str += "\n" # Show metadata for the first profile (i.e. the dynamic shapes) input_metadata = get_input_metadata_from_engine(engine, 0, bindings_per_profile) engine_str += "---- {:} Engine Input(s) ----\n{:}\n\n".format( len(input_metadata), input_metadata) output_metadata = get_output_metadata_from_engine(engine, 0, bindings_per_profile) engine_str += "---- {:} Engine Output(s) ----\n{:}\n\n".format( len(output_metadata), output_metadata) engine_str += "---- Memory ----\nDevice Memory: {:} bytes\n\n".format( engine.device_memory_size) engine_str += "---- {:} Profile(s) ({:} Binding(s) Each) ----\n".format( engine.num_optimization_profiles, bindings_per_profile) for profile_index in range(engine.num_optimization_profiles): engine_str += "- Profile: {:}\n".format(profile_index) max_width = max([len(binding) for binding in engine]) + 8 for offset in range(bindings_per_profile): binding = profile_index * bindings_per_profile + offset name = "[Name: {:}]".format(engine.get_binding_name(binding)) engine_str += util.indent_block( "Binding Index: {:} {:} {:<{max_width}}".format( binding, "(Input) " if engine.binding_is_input(binding) else "(Output)", name, max_width=max_width)) if engine.binding_is_input(binding): if engine.is_shape_binding(binding): min_shape, opt_shape, max_shape = engine.get_profile_shape_input( profile_index, binding) else: min_shape, opt_shape, max_shape = engine.get_profile_shape( profile_index, binding) engine_str += " | Shapes: min={:}, opt={:}, max={:}\n".format( min_shape, opt_shape, max_shape) else: engine_str += " | Shape: {:}".format( tuple(output_metadata[engine[offset]].shape)) engine_str += "\n" return util.indent_block(engine_str, level=0)
def str_from_network(network, mode="full"): """ Converts a TensorRT network to a human-readable representation Args: network (trt.INetworkDefinition): The network. mode (str): Controls what is displayed for each layer. Choices: ["none", "basic", "attrs", "full"] Returns: str """ LAYER_TYPE_CLASS_MAPPING = get_layer_class_mapping() network_str = "Name: {:} | {:} Batch Network{:}\n".format( network.name, "Implicit" if hasattr(network, "has_implicit_batch_dimension") and network.has_implicit_batch_dimension else "Explicit", " with Explicit Precision " if hasattr(network, "has_explicit_precision") and network.has_explicit_precision else "", ) network_str += "\n" input_metadata = get_network_input_metadata(network) network_str += "---- {:} Network Input(s) ----\n{:}\n\n".format(len(input_metadata), input_metadata) output_metadata = get_network_output_metadata(network) network_str += "---- {:} Network Output(s) ----\n{:}\n\n".format(len(output_metadata), output_metadata) network_str += "---- {:} Layer(s) ----\n".format(network.num_layers) if mode != "none": for index, layer in enumerate(network): if layer.type in LAYER_TYPE_CLASS_MAPPING: layer.__class__ = LAYER_TYPE_CLASS_MAPPING[layer.type] network_str += str_from_layer(layer, index) if mode in ["attrs", "full"]: # Exclude special attributes, as well as any attributes of the base layer class (those can be displayed above). attrs = get_layer_attribute_names(layer) if attrs: network_str += util.indent_block("---- Attributes ----") + "\n" for attr in attrs: with G_LOGGER.verbosity(): val = getattr(layer, attr) if mode == "full" or not isinstance(val, np.ndarray): attr_str = "" if layer.name: attr_str += "{:}.".format(layer.name) network_str += util.indent_block("{:}{:} = {:}".format(attr_str, attr, val)) + "\n" network_str += "\n" return util.indent_block(network_str, level=0)
def display_inputs(input_data): inputs_str = "" inputs_str += "==== Data ({:} iterations) ====\n".format( len(input_data)) inputs_str += str_from_iters(input_data) + "\n" inputs_str = util.indent_block(inputs_str, level=0).strip() G_LOGGER.info(inputs_str)
def display_results(results): results_str = "" results_str += "==== Run Results ({:} runners) ====\n\n".format( len(results)) for runner_name, iters in results.items(): results_str += "---- {:35} ({:} iterations) ----\n".format( runner_name, len(iters)) results_str += str_from_iters(iters) + "\n" results_str = util.indent_block(results_str, level=0).strip() G_LOGGER.info(results_str)
def execute_runner(runner, loader_cache): with runner as active_runner: input_metadata = active_runner.get_input_metadata() G_LOGGER.info("{:35}\n---- Model Input(s) ----\n{:}".format(active_runner.name, input_metadata), mode=LogMode.ONCE) # DataLoaderCache will ensure that the feed_dict does not contain any extra entries # based on the provided input_metadata. loader_cache.set_input_metadata(input_metadata) if warm_up: G_LOGGER.start("{:35} | Running {:} warm-up run(s)".format(active_runner.name, warm_up)) try: feed_dict = loader_cache[0] except IndexError: G_LOGGER.warning("{:} warm-up run(s) were requested, but data loader did not supply any data. " "Skipping warm-up run(s)".format(warm_up)) else: G_LOGGER.ultra_verbose("Warm-up Input Buffers:\n{:}".format(util.indent_block(feed_dict))) # First do a few warm-up runs, and don't time them. for _ in range(warm_up): active_runner.infer(feed_dict=feed_dict) G_LOGGER.finish("{:35} | Finished {:} warm-up run(s)".format(active_runner.name, warm_up)) # Then, actual iterations. index = 0 iteration_results = [] total_runtime = 0 for index, feed_dict in enumerate(loader_cache): G_LOGGER.extra_verbose(lambda: "{:35} | Feeding inputs:\n{:}".format(active_runner.name, util.indent_block(feed_dict))) outputs = active_runner.infer(feed_dict=feed_dict) runtime = active_runner.last_inference_time() total_runtime += runtime # Without a deep copy here, outputs will always reference the output of the last run iteration_results.append(IterationResult(outputs=copy.deepcopy(outputs), runtime=runtime, runner_name=active_runner.name)) G_LOGGER.info(lambda: "{:35}\n---- Model Output(s) ----\n{:}".format( active_runner.name, TensorMetadata().from_feed_dict(outputs)), mode=LogMode.ONCE) G_LOGGER.extra_verbose(lambda: "{:35} | Inference Time: {:.3f} ms | Received outputs:\n{:}".format( active_runner.name, runtime * 1000.0, util.indent_block(outputs))) total_runtime_ms = total_runtime * 1000.0 G_LOGGER.finish("{:35} | Completed {:} iteration(s) in {:.4g} ms | Average inference time: {:.4g} ms.".format(active_runner.name, index + 1, total_runtime_ms, total_runtime_ms / float(index + 1))) return iteration_results
def str_from_graph(graph, mode): graph_str = "" input_metadata = get_input_metadata(graph) output_metadata = get_output_metadata(graph) graph_str += "---- {:} Graph Inputs ----\n{:}\n\n".format( len(input_metadata), input_metadata) graph_str += "---- {:} Graph Outputs ----\n{:}\n\n".format( len(output_metadata), output_metadata) graph_str += "---- {:} Nodes ----\n".format(len(graph.as_graph_def().node)) if mode == "basic": G_LOGGER.warning( "Displaying layer information is unsupported for TensorFlow graphs. " "Please use --mode=full if you would like to see the raw nodes") if mode == "full": for node in graph.as_graph_def().node: graph_str += str(node) + "\n" graph_str += "\n" return util.indent_block(graph_str, level=0)
def process_attr(attr_str: str): processed = getattr(attr, ONNX_PYTHON_ATTR_MAPPING[attr_str]) if attr_str == "STRING": processed = processed.decode() elif attr_str == "TENSOR": tensor_str = "Tensor: [dtype={:}, shape={:}]".format( get_dtype(processed), get_shape(processed)) if mode == "full": tensor_str += " | Values:\n" + util.indent_block( str(get_values(processed))) processed = tensor_str elif attr_str == "GRAPH": processed = "\n" + str_from_onnx_graph( processed, mode, tensors, indent_level=indent_level + 2) elif attr_str == "FLOATS" or attr_str == "INTS": # Proto hacky list to normal Python list processed = [p for p in processed] elif attr_str == "STRINGS": processed = [p.decode() for p in processed] return processed
def check_outputs_match(out0, out0_name, out1, out1_name, per_out_rtol, per_out_atol, per_out_err_stat): VALID_CHECK_ERROR_STATS = ["max", "mean", "median", "elemwise"] if per_out_err_stat not in VALID_CHECK_ERROR_STATS: G_LOGGER.critical("Invalid choice for check_error_stat: {:}.\n" "Note: Valid choices are: {:}".format(per_out_err_stat, VALID_CHECK_ERROR_STATS)) G_LOGGER.super_verbose("{:35} | Output: {:} (dtype={:}, shape={:}):\n{:}".format( iter_result0.runner_name, out0_name, out0.dtype, out0.shape, util.indent_block(out0))) G_LOGGER.super_verbose("{:35} | Output: {:} (dtype={:}, shape={:}):\n{:}".format( iter_result1.runner_name, out1_name, out1.dtype, out1.shape, util.indent_block(out1))) # Check difference vs. tolerances if np.issubdtype(out0.dtype, np.bool_) and np.issubdtype(out1.dtype, np.bool_): absdiff = np.logical_xor(out0, out1) else: absdiff = np.abs(out0 - out1) absout1 = np.abs(out1) with np.testing.suppress_warnings() as sup: sup.filter(RuntimeWarning) reldiff = absdiff / absout1 max_absdiff = comp_util.compute_max(absdiff) mean_absdiff = comp_util.compute_mean(absdiff) median_absdiff = comp_util.compute_median(absdiff) max_reldiff = comp_util.compute_max(reldiff) mean_reldiff = comp_util.compute_mean(reldiff) median_reldiff = comp_util.compute_median(reldiff) max_elemwiseabs = "Unknown" max_elemwiserel = "Unknown" if per_out_err_stat == "mean": failed = mean_absdiff > per_out_atol and (np.isnan(mean_reldiff) or mean_reldiff > per_out_rtol) elif per_out_err_stat == "median": failed = median_absdiff > per_out_atol and (np.isnan(median_reldiff) or median_reldiff > per_out_rtol) elif per_out_err_stat == "max": failed = max_absdiff > per_out_atol and (np.isnan(max_reldiff) or max_reldiff > per_out_rtol) else: assert per_out_err_stat == "elemwise", "This branch should be unreachable unless per_out_err_stat is 'elemwise'" mismatches = (absdiff > per_out_atol) & (reldiff > per_out_rtol) failed = np.any(mismatches) try: # Special because we need to account for tolerances too. max_elemwiseabs = comp_util.compute_max(absdiff[mismatches]) max_elemwiserel = comp_util.compute_max(reldiff[mismatches]) with G_LOGGER.indent(): G_LOGGER.super_verbose("Mismatched indices:\n{:}".format(np.argwhere(mismatches))) G_LOGGER.extra_verbose("{:35} | Mismatched values:\n{:}".format(iter_result0.runner_name, out0[mismatches])) G_LOGGER.extra_verbose("{:35} | Mismatched values:\n{:}".format(iter_result1.runner_name, out1[mismatches])) except Exception as err: G_LOGGER.warning("Failing to log mismatches.\nNote: Error was: {:}".format(err)) # Log information about the outputs hist_bin_range = (min(comp_util.compute_min(out0), comp_util.compute_min(out1)), max(comp_util.compute_max(out0), comp_util.compute_max(out1))) comp_util.log_output_stats(out0, failed, iter_result0.runner_name + ": " + out0_name, hist_range=hist_bin_range) comp_util.log_output_stats(out1, failed, iter_result1.runner_name + ": " + out1_name, hist_range=hist_bin_range) G_LOGGER.info("Error Metrics: {:}".format(out0_name)) with G_LOGGER.indent(): def req_tol(mean_diff, median_diff, max_diff, elemwise_diff): return { "mean": mean_diff, "median": median_diff, "max": max_diff, "elemwise": elemwise_diff, }[per_out_err_stat] G_LOGGER.info("Minimum Required Tolerance: {:} error | [abs={:.5g}] OR [rel={:.5g}]".format( per_out_err_stat, req_tol(mean_absdiff, median_absdiff, max_absdiff, max_elemwiseabs), req_tol(mean_reldiff, median_reldiff, max_reldiff, max_elemwiserel))) comp_util.log_output_stats(absdiff, failed, "Absolute Difference") comp_util.log_output_stats(reldiff, failed, "Relative Difference") # Finally show summary. if failed: G_LOGGER.error("FAILED | Difference exceeds tolerance (rel={:}, abs={:})".format(per_out_rtol, per_out_atol)) else: G_LOGGER.finish("PASSED | Difference is within tolerance (rel={:}, abs={:})".format(per_out_rtol, per_out_atol)) G_LOGGER.extra_verbose("Finished comparing: '{:}' (dtype={:}, shape={:}) [{:}] and '{:}' (dtype={:}, shape={:}) [{:}]" .format(out0_name, out0.dtype, out0.shape, iter_result0.runner_name, out1_name, out1.dtype, out1.shape, iter_result1.runner_name)) return OutputCompareResult(not failed, max_absdiff, max_reldiff, mean_absdiff, mean_reldiff, median_absdiff, median_reldiff)
def str_from_onnx_graph(graph, mode, tensors, indent_level=0): input_metadata = get_input_metadata(graph) output_metadata = get_output_metadata(graph) initializer_metadata = get_tensor_metadata(graph.initializer) # Subgraph inputs should remain separate from each other, hence copy the tensors map tensors = copy.copy(tensors) tensors.update(get_tensor_metadata(graph.value_info)) tensors.update(initializer_metadata) tensors.update(input_metadata) tensors.update(output_metadata) graph_type = "Graph" if indent_level == 0 else "Subgraph" onnx_str = "" onnx_str += "---- {:} {:} Input(s) ----\n{:}\n\n".format( len(input_metadata), graph_type, input_metadata) onnx_str += "---- {:} {:} Output(s) ----\n{:}\n\n".format( len(output_metadata), graph_type, output_metadata) onnx_str += "---- {:} Initializer(s) ----\n".format( len(initializer_metadata)) if mode == "full": for init in graph.initializer: onnx_str += "Initializer | {:} [dtype={:}, shape={:}] | Values:\n{:}\n\n".format( init.name, get_dtype(init), get_shape(init), util.indent_block(str(get_values(init)))) if not graph.initializer: onnx_str += "{}\n\n" elif mode != "none": onnx_str += str(initializer_metadata) onnx_str += "\n\n" else: onnx_str += "\n" def metadata_from_names(names): metadata = TensorMetadata() for name in names: dtype, shape = tensors.get(name, (None, None)) if name in initializer_metadata: name = "Initializer | {:}".format(name) metadata.add(name=name, dtype=dtype, shape=shape) return metadata # Maps values from the AttributeType enum to their string representations, e.g., {1: "FLOAT"} ATTR_TYPE_MAPPING = dict( zip(onnx.AttributeProto.AttributeType.values(), onnx.AttributeProto.AttributeType.keys())) # Maps an ONNX attribute to the corresponding Python property ONNX_PYTHON_ATTR_MAPPING = { "FLOAT": "f", "INT": "i", "STRING": "s", "TENSOR": "t", "GRAPH": "g", "FLOATS": "floats", "INTS": "ints", "STRINGS": "strings", } def attrs_to_dict(attrs): attr_dict = OrderedDict() for attr in attrs: def process_attr(attr_str: str): processed = getattr(attr, ONNX_PYTHON_ATTR_MAPPING[attr_str]) if attr_str == "STRING": processed = processed.decode() elif attr_str == "TENSOR": tensor_str = "Tensor: [dtype={:}, shape={:}]".format( get_dtype(processed), get_shape(processed)) if mode == "full": tensor_str += " | Values:\n" + util.indent_block( str(get_values(processed))) processed = tensor_str elif attr_str == "GRAPH": processed = "\n" + str_from_onnx_graph( processed, mode, tensors, indent_level=indent_level + 2) elif attr_str == "FLOATS" or attr_str == "INTS": # Proto hacky list to normal Python list processed = [p for p in processed] elif attr_str == "STRINGS": processed = [p.decode() for p in processed] return processed if attr.type in ATTR_TYPE_MAPPING: attr_str = ATTR_TYPE_MAPPING[attr.type] if attr_str in ONNX_PYTHON_ATTR_MAPPING: attr_dict[attr.name] = process_attr(attr_str) else: G_LOGGER.warning( "Attribute of type {:} is currently unsupported. Skipping attribute." .format(attr_str)) else: G_LOGGER.warning( "Attribute type: {:} was not recognized. Was the graph generated with a newer IR " "version than the installed `onnx` package? Skipping attribute." .format(attr.type)) return attr_dict onnx_str += "---- {:} Node(s) ----\n".format(len(graph.node)) if mode != "none": for index, node in enumerate(graph.node): input_info = metadata_from_names(node.input) output_info = metadata_from_names(node.output) onnx_str += util.str_from_layer("Node", index, node.name, node.op_type, input_info, output_info) if mode in ["attrs", "full"]: attrs = attrs_to_dict(node.attribute) if attrs: onnx_str += util.indent_block( "---- Attributes ----") + "\n" for key, val in attrs.items(): attr_str = "" if node.name: attr_str += "{:}.".format(node.name) onnx_str += util.indent_block("{:}{:} = {:}".format( attr_str, key, val)) + "\n" onnx_str += "\n" return util.indent_block(onnx_str, indent_level)
def str_from_network(network, mode="full"): """ Converts a TensorRT network to a human-readable representation Args: network (trt.INetworkDefinition): The network. mode (str): Controls what is displayed for each layer. Choices: ["none", "basic", "attrs", "full"] Returns: str """ LAYER_TYPE_CLASS_MAPPING = get_layer_class_mapping() def is_special_attribute(attr): return attr.startswith("__") and attr.endswith("__") def is_valid_attribute(attr, layer): if type(layer) == trt.IPoolingLayer or type( layer) == trt.IConvolutionLayer or type( layer) == trt.IDeconvolutionLayer: if len(layer.get_input(0).shape) > 4: # 3D pooling uses padding_nd return attr not in ["padding", "stride", "window_size"] if type(layer) == trt.IResizeLayer: if layer.num_inputs > 1: return attr not in ["scales"] if type(layer) == trt.ISliceLayer: if layer.num_inputs > 1: return attr not in ["shape", "start", "stride"] return True network_str = "Name: {:} | {:} Batch Network{:}\n".format( network.name, "Implicit" if hasattr(network, "has_implicit_batch_dimension") and network.has_implicit_batch_dimension else "Explicit", " with Explicit Precision " if hasattr(network, "has_explicit_precision") and network.has_explicit_precision else "") network_str += "\n" input_metadata = get_input_metadata(network) network_str += "---- {:} Network Input(s) ----\n{:}\n\n".format( len(input_metadata), input_metadata) output_metadata = get_output_metadata(network) network_str += "---- {:} Network Output(s) ----\n{:}\n\n".format( len(output_metadata), output_metadata) network_str += "---- {:} Layer(s) ----\n".format(network.num_layers) if mode != "none": for index, layer in enumerate(network): if layer.type in LAYER_TYPE_CLASS_MAPPING: layer.__class__ = LAYER_TYPE_CLASS_MAPPING[layer.type] network_str += str_from_layer(layer, index) if mode in ["attrs", "full"]: # Exclude special attributes, as well as any attributes of the base layer class (those can be displayed above). attrs = [ attr for attr in dir(layer) if not is_special_attribute(attr) and not hasattr( trt.ILayer, attr) and is_valid_attribute(attr, layer) ] if attrs: network_str += util.indent_block( "---- Attributes ----") + "\n" for attr in attrs: with G_LOGGER.verbosity(): val = getattr(layer, attr) if mode == "full" or not isinstance(val, np.ndarray): attr_str = "" if layer.name: attr_str += "{:}.".format(layer.name) network_str += util.indent_block("{:}{:} = {:}".format( attr_str, attr, val)) + "\n" network_str += "\n" return util.indent_block(network_str, level=0)