def get_layer_output_metadata(layer):
meta = TensorMetadata()
for i in range(layer.num_outputs):
outp = layer.get_output(i)
if outp:
meta.add(outp.name, np_dtype_from_trt(outp.dtype), outp.shape)
return meta
def get_layer_input_metadata(layer):
meta = TensorMetadata()
for i in range(layer.num_inputs):
inp = layer.get_input(i)
if inp:
meta.add(inp.name, np_dtype_from_trt(inp.dtype), inp.shape)
return meta
def get_tensor_metadata(tensors):
metadata = TensorMetadata()
for tensor in tensors:
metadata.add(name=tensor.name,
dtype=get_dtype(tensor),
shape=get_shape(tensor))
return metadata
def get_input_metadata(self):
ONNX_RT_TYPE_TO_NP = {
"tensor(double)": np.float64,
"tensor(float)": np.float32,
"tensor(float16)": np.float16,
"tensor(int16)": np.int16,
"tensor(int32)": np.int32,
"tensor(int64)": np.int64,
"tensor(int8)": np.int8,
"tensor(uint16)": np.uint16,
"tensor(uint32)": np.uint32,
"tensor(uint64)": np.uint64,
"tensor(uint8)": np.uint8,
"tensor(bool)": np.bool,
}
def process_shape(shape):
# dims can be strings to indicate dynamic dimensions. Polygraphy uses None for the same purpose
return [
None if not isinstance(elem, int) else elem for elem in shape
]
meta = TensorMetadata()
for node in self.sess.get_inputs():
meta.add(node.name,
dtype=ONNX_RT_TYPE_TO_NP[node.type],
shape=process_shape(node.shape))
return meta
def get_input_metadata_from_profile(profile, network):
"""
Returns metadata about the inputs based on the OPT values set in a profile.
Args:
profile (trt.IOptimizationProfile):
The profile from which to retrieve input metada.
network (trt.INetworkDefinition):
The network the profile applies to.
Returns:
TensorMetadata:
A mapping of input names to their types and shapes.
Shapes are retrieved from the OPT values in the profile.
"""
input_metadata = TensorMetadata()
for index in range(network.num_inputs):
tensor = network.get_input(index)
if tensor.is_shape_tensor:
shapes = profile.get_shape_input(tensor.name)
else:
shapes = profile.get_shape(tensor.name)
if tuple(shapes[0]) != tuple(shapes[2]):
G_LOGGER.warning(
"Will use `opt` shapes from profile 0 for calibration. "
"Note that even though `min` != `max` in this profile, calibration "
"will use fixed input shapes (this is not necessarily an issue)."
)
# Always use opt shape
input_metadata.add(name=tensor.name,
dtype=trt.nptype(tensor.dtype),
shape=shapes[1])
return input_metadata
def _input_metadata_from_network(network):
input_metadata = TensorMetadata()
for index in range(network.num_inputs):
tensor = network.get_input(index)
input_metadata.add(name=tensor.name,
dtype=np.dtype(trt.nptype(tensor.dtype)),
shape=tensor.shape)
return input_metadata
def metadata_from_names(names):
metadata = TensorMetadata()
for name in names:
dtype, shape = tensors.get(name, (None, None))
if name in initializer_metadata:
name = "Initializer | {:}".format(name)
metadata.add(name=name, dtype=dtype, shape=shape)
return metadata
def get_output_metadata(network):
outputs = TensorMetadata()
for i in range(network.num_outputs):
tensor = network.get_output(i)
outputs.add(name=tensor.name,
dtype=np_dtype_from_trt(tensor.dtype),
shape=tensor.shape)
return outputs
def missing_meta_tensors(input_metadata, output_metadata):
missing = TensorMetadata()
for name, (dtype, shape) in input_metadata.items():
if dtype is None or shape is None:
missing.add(name, dtype, shape)
for name, (dtype, shape) in output_metadata.items():
if dtype is None:
missing.add(name, dtype, shape)
return missing
def get_input_metadata_impl(self):
inputs = TensorMetadata()
for binding in self.engine:
if self.engine.binding_is_input(binding):
# Always prepend a dynamic batch dimension
inputs.add(binding,
trt.nptype(self.engine.get_binding_dtype(binding)),
[-1] + list(self.engine.get_binding_shape(binding)))
return inputs
def get_tensor_metadata(tensors):
metadata = TensorMetadata()
for tensor in tensors:
try:
shape = [elem.value if hasattr(elem, "value") else elem for elem in tensor.shape]
except ValueError:
# Happens when rank is unknown
shape = None
metadata.add(tensor.name, dtype=tensor.dtype.as_numpy_dtype, shape=shape)
return metadata
def fallback_shape_inference(onnx_model):
from polygraphy.backend.onnx import BytesFromOnnx, ModifyOnnx
from polygraphy.backend.onnxrt import (OnnxrtRunner,
SessionFromOnnxBytes)
load_model = ModifyOnnx(onnx_model, outputs=constants.MARK_ALL)
with OnnxrtRunner(SessionFromOnnxBytes(BytesFromOnnx(load_model))) as runner:
data_loader = self.makers[DataLoaderArgs].get_data_loader()
data_loader.input_metadata = runner.get_input_metadata()
outputs = runner.infer(feed_dict=data_loader[0])
meta = TensorMetadata()
for name, output in outputs.items():
meta.add(name, output.dtype, output.shape)
return meta
def get_input_metadata_impl(self):
ONNX_RT_TYPE_TO_NP = {
"tensor(double)": np.float64,
"tensor(float)": np.float32,
"tensor(float16)": np.float16,
"tensor(int16)": np.int16,
"tensor(int32)": np.int32,
"tensor(int64)": np.int64,
"tensor(int8)": np.int8,
"tensor(uint16)": np.uint16,
"tensor(uint32)": np.uint32,
"tensor(uint64)": np.uint64,
"tensor(uint8)": np.uint8,
"tensor(bool)": np.bool,
"tensor(string)": np.unicode,
}
meta = TensorMetadata()
for node in self.sess.get_inputs():
dtype = ONNX_RT_TYPE_TO_NP[node.type] if node.type in ONNX_RT_TYPE_TO_NP else None
meta.add(node.name, dtype=dtype, shape=node.shape)
return meta
def parse_meta_new_impl(meta_args, includes_shape=True, includes_dtype=True):
SEP = ":"
meta = TensorMetadata()
for meta_arg in meta_args:
name, shape, dtype = None, None, None
def pop_meta(func):
nonlocal meta_arg
meta_arg, _, val = meta_arg.rpartition(SEP)
val = cast(val.strip())
if isinstance(val, str) and val.lower() == "auto":
return None
return func(val)
if includes_dtype:
dtype = pop_meta(func=np_type_from_str)
if includes_shape:
shape = pop_meta(func=lambda s: tuple(e for e in s if e != ""))
name = meta_arg
meta.add(name, dtype=dtype, shape=shape)
return meta
def get_input_metadata_from_profile(profile, network):
"""
Returns metadata about the inputs based on a profile
Args:
profile (trt.IOptimizationProfile): The profile from which to retrieve input metada.
network (trt.INetworkDefinition): The network
Returns:
TensorMetadata: A mapping of input names to their types and shapes.
"""
input_metadata = TensorMetadata()
for index in range(network.num_inputs):
tensor = network.get_input(index)
if tensor.is_shape_tensor:
shapes = profile.get_shape_input(tensor.name)
else:
shapes = profile.get_shape(tensor.name)
if tuple(shapes[0]) != tuple(shapes[1]):
G_LOGGER.warning("In profile 0, min != max, using opt shapes for calibration")
# Always use opt shape
input_metadata.add(name=tensor.name, dtype=trt.nptype(tensor.dtype), shape=shapes[1])
return input_metadata
def meta_from_iter_result(iter_result):
meta = TensorMetadata()
for name, arr in iter_result.items():
meta.add(name, dtype=arr.dtype, shape=arr.shape)
return meta
def meta_from_gs_tensors(tensors):
"""Get TensorMetadata from a list of ONNX-GraphSurgeon tensors"""
meta = TensorMetadata()
for tensor in tensors:
meta.add(tensor.name, tensor.dtype, tensor.shape)
return meta
def get_input_metadata(network):
inputs = TensorMetadata()
for i in range(network.num_inputs):
tensor = network.get_input(i)
inputs.add(name=tensor.name, dtype=trt.nptype(tensor.dtype), shape=tensor.shape)
return inputs
def parse_meta_legacy(meta_args, includes_shape=True, includes_dtype=True):
"""
Parses a list of tensor metadata arguments of the form "<name>,<shape>,<dtype>"
`shape` and `dtype` are optional, but `dtype` must always come after `shape` if they are both enabled.
Args:
meta_args (List[str]): A list of tensor metadata arguments from the command-line.
includes_shape (bool): Whether the arguments include shape information.
includes_dtype (bool): Whether the arguments include dtype information.
Returns:
TensorMetadata: The parsed tensor metadata.
"""
SEP = ","
SHAPE_SEP = "x"
meta = TensorMetadata()
for orig_tensor_meta_arg in meta_args:
tensor_meta_arg = orig_tensor_meta_arg
def pop_meta(name):
nonlocal tensor_meta_arg
tensor_meta_arg, _, val = tensor_meta_arg.rpartition(SEP)
if not tensor_meta_arg:
G_LOGGER.exit("Could not parse {:} from argument: {:}. Is it separated by a comma "
"(,) from the tensor name?".format(name, orig_tensor_meta_arg))
if val.lower() == "auto":
val = None
return val
def parse_dtype(dtype):
if dtype is not None:
dtype = np_type_from_str(dtype)
return dtype
def parse_shape(shape):
if shape is not None:
def parse_shape_dim(buf):
try:
buf = int(buf)
except:
pass
return buf
parsed_shape = []
# Allow for quoted strings in shape dimensions
in_quotes = False
buf = ""
for char in shape.lower():
if char in ["\"", "'"]:
in_quotes = not in_quotes
elif not in_quotes and char == SHAPE_SEP:
parsed_shape.append(parse_shape_dim(buf))
buf = ""
else:
buf += char
# For the last dimension
if buf:
parsed_shape.append(parse_shape_dim(buf))
shape = tuple(parsed_shape)
return shape
name = None
dtype = None
shape = None
if includes_dtype:
dtype = parse_dtype(pop_meta("data type"))
if includes_shape:
shape = parse_shape(pop_meta("shape"))
name = tensor_meta_arg
meta.add(name, dtype, shape)
return meta
def meta_from_tensors(tensors):
meta = TensorMetadata()
for tensor in tensors:
meta.add(tensor.name, tensor.dtype, tensor.shape)
return meta
def execute_runner(runner, loader_cache):
with runner as active_runner:
input_metadata = active_runner.get_input_metadata()
G_LOGGER.info("Runner: {:40} | Input Metadata: {:}".format(
active_runner.name, input_metadata),
mode=LogMode.ONCE)
# DataLoaderCache will ensure that the feed_dict does not contain any extra entries
# based on the provided input_metadata.
loader_cache.set_input_metadata(input_metadata)
if warm_up:
G_LOGGER.start(
"Runner: {:40} | Running {:} warm-up runs".format(
active_runner.name, warm_up))
try:
feed_dict = loader_cache[0]
except IndexError:
G_LOGGER.warning(
"{:} warm-up runs were requested, but data loader did not supply any data. "
"Skipping warm-up runs".format(warm_up))
else:
G_LOGGER.ultra_verbose(
"Warm-up Input Buffers:\n{:}".format(
misc.indent_block(feed_dict)))
# First do a few warm-up runs, and don't time them.
for i in range(warm_up):
active_runner.infer(feed_dict=feed_dict)
# Then, actual iterations.
index = 0
iteration_results = []
output_metadata = TensorMetadata()
for index, feed_dict in enumerate(loader_cache):
G_LOGGER.extra_verbose(
lambda: "Runner: {:40} | Feeding inputs:\n{:}".format(
active_runner.name, misc.indent_block(feed_dict)))
outputs = active_runner.infer(feed_dict=feed_dict)
runtime = active_runner.last_inference_time()
# Without a deep copy here, outputs will always reference the output of the last run
iteration_results.append(
IterationResult(outputs=copy.deepcopy(outputs),
runtime=runtime,
runner_name=active_runner.name))
if index == 0:
for name, out in outputs.items():
output_metadata.add(name, out.dtype, out.shape)
G_LOGGER.info(
"Runner: {:40} | Output Metadata: {:}".format(
active_runner.name, output_metadata),
mode=LogMode.ONCE)
G_LOGGER.extra_verbose(
lambda:
"Runner: {:40} | Inference Time: {:.3f} ms | Received outputs:\n{:}"
.format(active_runner.name, runtime * 1000.0,
misc.indent_block(outputs)))
G_LOGGER.finish(
"Runner: {:40} | Completed {:} iterations.".format(
active_runner.name, index + 1))
return iteration_results
def parse_meta_legacy(meta_args, includes_shape=True, includes_dtype=True):
"""
Parses a list of tensor metadata arguments of the form "<name>,<shape>,<dtype>"
`shape` and `dtype` are optional, but `dtype` must always come after `shape` if they are both enabled.
Args:
meta_args (List[str]): A list of tensor metadata arguments from the command-line.
includes_shape (bool): Whether the arguments include shape information.
includes_dtype (bool): Whether the arguments include dtype information.
Returns:
TensorMetadata: The parsed tensor metadata.
"""
SEP = ","
SHAPE_SEP = "x"
meta = TensorMetadata()
for orig_tensor_meta_arg in meta_args:
tensor_meta_arg = orig_tensor_meta_arg
def pop_meta(name):
nonlocal tensor_meta_arg
tensor_meta_arg, _, val = tensor_meta_arg.rpartition(SEP)
if not tensor_meta_arg:
G_LOGGER.critical(
"Could not parse {:} from argument: {:}. Is it separated by a comma "
"(,) from the tensor name?".format(name,
orig_tensor_meta_arg))
if val.lower() == "auto":
val = None
return val
def parse_dtype(dtype):
if dtype is not None:
dtype = np_type_from_str(dtype)
return dtype
def parse_shape(shape):
if shape is not None:
def parse_shape_dim(buf):
try:
buf = int(buf)
except:
pass
return buf
parsed_shape = []
# Allow for quoted strings in shape dimensions
in_quotes = False
buf = ""
for char in shape.lower():
if char in ['"', "'"]:
in_quotes = not in_quotes
elif not in_quotes and char == SHAPE_SEP:
parsed_shape.append(parse_shape_dim(buf))
buf = ""
else:
buf += char
# For the last dimension
if buf:
parsed_shape.append(parse_shape_dim(buf))
shape = tuple(parsed_shape)
return shape
name = None
dtype = None
shape = None
if includes_dtype:
dtype = parse_dtype(pop_meta("data type"))
if includes_shape:
shape = parse_shape(pop_meta("shape"))
name = tensor_meta_arg
meta.add(name, dtype, shape)
new_style = []
for m_arg in meta_args:
arg = m_arg
if includes_shape:
arg = arg.replace(",", ":[", 1)
if includes_dtype:
arg = arg.replace(",", "]:", 1)
else:
arg += "]"
arg = arg.replace(",auto", ":auto")
arg = arg.replace(",", ":")
if includes_shape:
arg = arg.replace("x", ",")
new_style.append(arg)
G_LOGGER.warning(
"The old shape syntax is deprecated and will be removed in a future version of Polygraphy\n"
"See the CHANGELOG for the motivation behind this deprecation.",
mode=LogMode.ONCE,
)
G_LOGGER.warning("Instead of: '{:}', use: '{:}'\n".format(
" ".join(meta_args), " ".join(new_style)))
return meta
