def get_layer_output_metadata(layer): meta = TensorMetadata() for i in range(layer.num_outputs): outp = layer.get_output(i) if outp: meta.add(outp.name, np_dtype_from_trt(outp.dtype), outp.shape) return meta
def get_layer_input_metadata(layer): meta = TensorMetadata() for i in range(layer.num_inputs): inp = layer.get_input(i) if inp: meta.add(inp.name, np_dtype_from_trt(inp.dtype), inp.shape) return meta
def get_tensor_metadata(tensors): metadata = TensorMetadata() for tensor in tensors: metadata.add(name=tensor.name, dtype=get_dtype(tensor), shape=get_shape(tensor)) return metadata
def get_input_metadata(self): ONNX_RT_TYPE_TO_NP = { "tensor(double)": np.float64, "tensor(float)": np.float32, "tensor(float16)": np.float16, "tensor(int16)": np.int16, "tensor(int32)": np.int32, "tensor(int64)": np.int64, "tensor(int8)": np.int8, "tensor(uint16)": np.uint16, "tensor(uint32)": np.uint32, "tensor(uint64)": np.uint64, "tensor(uint8)": np.uint8, "tensor(bool)": np.bool, } def process_shape(shape): # dims can be strings to indicate dynamic dimensions. Polygraphy uses None for the same purpose return [ None if not isinstance(elem, int) else elem for elem in shape ] meta = TensorMetadata() for node in self.sess.get_inputs(): meta.add(node.name, dtype=ONNX_RT_TYPE_TO_NP[node.type], shape=process_shape(node.shape)) return meta
def get_input_metadata_from_profile(profile, network): """ Returns metadata about the inputs based on the OPT values set in a profile. Args: profile (trt.IOptimizationProfile): The profile from which to retrieve input metada. network (trt.INetworkDefinition): The network the profile applies to. Returns: TensorMetadata: A mapping of input names to their types and shapes. Shapes are retrieved from the OPT values in the profile. """ input_metadata = TensorMetadata() for index in range(network.num_inputs): tensor = network.get_input(index) if tensor.is_shape_tensor: shapes = profile.get_shape_input(tensor.name) else: shapes = profile.get_shape(tensor.name) if tuple(shapes[0]) != tuple(shapes[2]): G_LOGGER.warning( "Will use `opt` shapes from profile 0 for calibration. " "Note that even though `min` != `max` in this profile, calibration " "will use fixed input shapes (this is not necessarily an issue)." ) # Always use opt shape input_metadata.add(name=tensor.name, dtype=trt.nptype(tensor.dtype), shape=shapes[1]) return input_metadata
def _input_metadata_from_network(network): input_metadata = TensorMetadata() for index in range(network.num_inputs): tensor = network.get_input(index) input_metadata.add(name=tensor.name, dtype=np.dtype(trt.nptype(tensor.dtype)), shape=tensor.shape) return input_metadata
def metadata_from_names(names): metadata = TensorMetadata() for name in names: dtype, shape = tensors.get(name, (None, None)) if name in initializer_metadata: name = "Initializer | {:}".format(name) metadata.add(name=name, dtype=dtype, shape=shape) return metadata
def get_output_metadata(network): outputs = TensorMetadata() for i in range(network.num_outputs): tensor = network.get_output(i) outputs.add(name=tensor.name, dtype=np_dtype_from_trt(tensor.dtype), shape=tensor.shape) return outputs
def missing_meta_tensors(input_metadata, output_metadata): missing = TensorMetadata() for name, (dtype, shape) in input_metadata.items(): if dtype is None or shape is None: missing.add(name, dtype, shape) for name, (dtype, shape) in output_metadata.items(): if dtype is None: missing.add(name, dtype, shape) return missing
def get_input_metadata_impl(self): inputs = TensorMetadata() for binding in self.engine: if self.engine.binding_is_input(binding): # Always prepend a dynamic batch dimension inputs.add(binding, trt.nptype(self.engine.get_binding_dtype(binding)), [-1] + list(self.engine.get_binding_shape(binding))) return inputs
def get_tensor_metadata(tensors): metadata = TensorMetadata() for tensor in tensors: try: shape = [elem.value if hasattr(elem, "value") else elem for elem in tensor.shape] except ValueError: # Happens when rank is unknown shape = None metadata.add(tensor.name, dtype=tensor.dtype.as_numpy_dtype, shape=shape) return metadata
def fallback_shape_inference(onnx_model): from polygraphy.backend.onnx import BytesFromOnnx, ModifyOnnx from polygraphy.backend.onnxrt import (OnnxrtRunner, SessionFromOnnxBytes) load_model = ModifyOnnx(onnx_model, outputs=constants.MARK_ALL) with OnnxrtRunner(SessionFromOnnxBytes(BytesFromOnnx(load_model))) as runner: data_loader = self.makers[DataLoaderArgs].get_data_loader() data_loader.input_metadata = runner.get_input_metadata() outputs = runner.infer(feed_dict=data_loader[0]) meta = TensorMetadata() for name, output in outputs.items(): meta.add(name, output.dtype, output.shape) return meta
def get_input_metadata_impl(self): ONNX_RT_TYPE_TO_NP = { "tensor(double)": np.float64, "tensor(float)": np.float32, "tensor(float16)": np.float16, "tensor(int16)": np.int16, "tensor(int32)": np.int32, "tensor(int64)": np.int64, "tensor(int8)": np.int8, "tensor(uint16)": np.uint16, "tensor(uint32)": np.uint32, "tensor(uint64)": np.uint64, "tensor(uint8)": np.uint8, "tensor(bool)": np.bool, "tensor(string)": np.unicode, } meta = TensorMetadata() for node in self.sess.get_inputs(): dtype = ONNX_RT_TYPE_TO_NP[node.type] if node.type in ONNX_RT_TYPE_TO_NP else None meta.add(node.name, dtype=dtype, shape=node.shape) return meta
def parse_meta_new_impl(meta_args, includes_shape=True, includes_dtype=True): SEP = ":" meta = TensorMetadata() for meta_arg in meta_args: name, shape, dtype = None, None, None def pop_meta(func): nonlocal meta_arg meta_arg, _, val = meta_arg.rpartition(SEP) val = cast(val.strip()) if isinstance(val, str) and val.lower() == "auto": return None return func(val) if includes_dtype: dtype = pop_meta(func=np_type_from_str) if includes_shape: shape = pop_meta(func=lambda s: tuple(e for e in s if e != "")) name = meta_arg meta.add(name, dtype=dtype, shape=shape) return meta
def get_input_metadata_from_profile(profile, network): """ Returns metadata about the inputs based on a profile Args: profile (trt.IOptimizationProfile): The profile from which to retrieve input metada. network (trt.INetworkDefinition): The network Returns: TensorMetadata: A mapping of input names to their types and shapes. """ input_metadata = TensorMetadata() for index in range(network.num_inputs): tensor = network.get_input(index) if tensor.is_shape_tensor: shapes = profile.get_shape_input(tensor.name) else: shapes = profile.get_shape(tensor.name) if tuple(shapes[0]) != tuple(shapes[1]): G_LOGGER.warning("In profile 0, min != max, using opt shapes for calibration") # Always use opt shape input_metadata.add(name=tensor.name, dtype=trt.nptype(tensor.dtype), shape=shapes[1]) return input_metadata
def meta_from_iter_result(iter_result): meta = TensorMetadata() for name, arr in iter_result.items(): meta.add(name, dtype=arr.dtype, shape=arr.shape) return meta
def meta_from_gs_tensors(tensors): """Get TensorMetadata from a list of ONNX-GraphSurgeon tensors""" meta = TensorMetadata() for tensor in tensors: meta.add(tensor.name, tensor.dtype, tensor.shape) return meta
def get_input_metadata(network): inputs = TensorMetadata() for i in range(network.num_inputs): tensor = network.get_input(i) inputs.add(name=tensor.name, dtype=trt.nptype(tensor.dtype), shape=tensor.shape) return inputs
def parse_meta_legacy(meta_args, includes_shape=True, includes_dtype=True): """ Parses a list of tensor metadata arguments of the form "<name>,<shape>,<dtype>" `shape` and `dtype` are optional, but `dtype` must always come after `shape` if they are both enabled. Args: meta_args (List[str]): A list of tensor metadata arguments from the command-line. includes_shape (bool): Whether the arguments include shape information. includes_dtype (bool): Whether the arguments include dtype information. Returns: TensorMetadata: The parsed tensor metadata. """ SEP = "," SHAPE_SEP = "x" meta = TensorMetadata() for orig_tensor_meta_arg in meta_args: tensor_meta_arg = orig_tensor_meta_arg def pop_meta(name): nonlocal tensor_meta_arg tensor_meta_arg, _, val = tensor_meta_arg.rpartition(SEP) if not tensor_meta_arg: G_LOGGER.exit("Could not parse {:} from argument: {:}. Is it separated by a comma " "(,) from the tensor name?".format(name, orig_tensor_meta_arg)) if val.lower() == "auto": val = None return val def parse_dtype(dtype): if dtype is not None: dtype = np_type_from_str(dtype) return dtype def parse_shape(shape): if shape is not None: def parse_shape_dim(buf): try: buf = int(buf) except: pass return buf parsed_shape = [] # Allow for quoted strings in shape dimensions in_quotes = False buf = "" for char in shape.lower(): if char in ["\"", "'"]: in_quotes = not in_quotes elif not in_quotes and char == SHAPE_SEP: parsed_shape.append(parse_shape_dim(buf)) buf = "" else: buf += char # For the last dimension if buf: parsed_shape.append(parse_shape_dim(buf)) shape = tuple(parsed_shape) return shape name = None dtype = None shape = None if includes_dtype: dtype = parse_dtype(pop_meta("data type")) if includes_shape: shape = parse_shape(pop_meta("shape")) name = tensor_meta_arg meta.add(name, dtype, shape) return meta
def meta_from_tensors(tensors): meta = TensorMetadata() for tensor in tensors: meta.add(tensor.name, tensor.dtype, tensor.shape) return meta
def execute_runner(runner, loader_cache): with runner as active_runner: input_metadata = active_runner.get_input_metadata() G_LOGGER.info("Runner: {:40} | Input Metadata: {:}".format( active_runner.name, input_metadata), mode=LogMode.ONCE) # DataLoaderCache will ensure that the feed_dict does not contain any extra entries # based on the provided input_metadata. loader_cache.set_input_metadata(input_metadata) if warm_up: G_LOGGER.start( "Runner: {:40} | Running {:} warm-up runs".format( active_runner.name, warm_up)) try: feed_dict = loader_cache[0] except IndexError: G_LOGGER.warning( "{:} warm-up runs were requested, but data loader did not supply any data. " "Skipping warm-up runs".format(warm_up)) else: G_LOGGER.ultra_verbose( "Warm-up Input Buffers:\n{:}".format( misc.indent_block(feed_dict))) # First do a few warm-up runs, and don't time them. for i in range(warm_up): active_runner.infer(feed_dict=feed_dict) # Then, actual iterations. index = 0 iteration_results = [] output_metadata = TensorMetadata() for index, feed_dict in enumerate(loader_cache): G_LOGGER.extra_verbose( lambda: "Runner: {:40} | Feeding inputs:\n{:}".format( active_runner.name, misc.indent_block(feed_dict))) outputs = active_runner.infer(feed_dict=feed_dict) runtime = active_runner.last_inference_time() # Without a deep copy here, outputs will always reference the output of the last run iteration_results.append( IterationResult(outputs=copy.deepcopy(outputs), runtime=runtime, runner_name=active_runner.name)) if index == 0: for name, out in outputs.items(): output_metadata.add(name, out.dtype, out.shape) G_LOGGER.info( "Runner: {:40} | Output Metadata: {:}".format( active_runner.name, output_metadata), mode=LogMode.ONCE) G_LOGGER.extra_verbose( lambda: "Runner: {:40} | Inference Time: {:.3f} ms | Received outputs:\n{:}" .format(active_runner.name, runtime * 1000.0, misc.indent_block(outputs))) G_LOGGER.finish( "Runner: {:40} | Completed {:} iterations.".format( active_runner.name, index + 1)) return iteration_results
def parse_meta_legacy(meta_args, includes_shape=True, includes_dtype=True): """ Parses a list of tensor metadata arguments of the form "<name>,<shape>,<dtype>" `shape` and `dtype` are optional, but `dtype` must always come after `shape` if they are both enabled. Args: meta_args (List[str]): A list of tensor metadata arguments from the command-line. includes_shape (bool): Whether the arguments include shape information. includes_dtype (bool): Whether the arguments include dtype information. Returns: TensorMetadata: The parsed tensor metadata. """ SEP = "," SHAPE_SEP = "x" meta = TensorMetadata() for orig_tensor_meta_arg in meta_args: tensor_meta_arg = orig_tensor_meta_arg def pop_meta(name): nonlocal tensor_meta_arg tensor_meta_arg, _, val = tensor_meta_arg.rpartition(SEP) if not tensor_meta_arg: G_LOGGER.critical( "Could not parse {:} from argument: {:}. Is it separated by a comma " "(,) from the tensor name?".format(name, orig_tensor_meta_arg)) if val.lower() == "auto": val = None return val def parse_dtype(dtype): if dtype is not None: dtype = np_type_from_str(dtype) return dtype def parse_shape(shape): if shape is not None: def parse_shape_dim(buf): try: buf = int(buf) except: pass return buf parsed_shape = [] # Allow for quoted strings in shape dimensions in_quotes = False buf = "" for char in shape.lower(): if char in ['"', "'"]: in_quotes = not in_quotes elif not in_quotes and char == SHAPE_SEP: parsed_shape.append(parse_shape_dim(buf)) buf = "" else: buf += char # For the last dimension if buf: parsed_shape.append(parse_shape_dim(buf)) shape = tuple(parsed_shape) return shape name = None dtype = None shape = None if includes_dtype: dtype = parse_dtype(pop_meta("data type")) if includes_shape: shape = parse_shape(pop_meta("shape")) name = tensor_meta_arg meta.add(name, dtype, shape) new_style = [] for m_arg in meta_args: arg = m_arg if includes_shape: arg = arg.replace(",", ":[", 1) if includes_dtype: arg = arg.replace(",", "]:", 1) else: arg += "]" arg = arg.replace(",auto", ":auto") arg = arg.replace(",", ":") if includes_shape: arg = arg.replace("x", ",") new_style.append(arg) G_LOGGER.warning( "The old shape syntax is deprecated and will be removed in a future version of Polygraphy\n" "See the CHANGELOG for the motivation behind this deprecation.", mode=LogMode.ONCE, ) G_LOGGER.warning("Instead of: '{:}', use: '{:}'\n".format( " ".join(meta_args), " ".join(new_style))) return meta