def _set_shapes_from_feed_dict(self, feed_dict):
"""
Sets context shapes according to the provided feed_dict.
Note that ``infer()`` will call this function automatically, and hence
you should only use it if you plan to use this runner's context manually.
Args:
feed_dict (OrderedDict[str, numpy.ndarray]):
A mapping of input tensor names to corresponding input NumPy arrays.
Returns:
Tuple[int, int]: The start and end binding indices of the modified bindings.
"""
def is_dynamic_shape_input(binding):
try:
self.context.engine.get_profile_shape_input(0, binding)
return True
except RuntimeError:
return False
start_binding, end_binding = trt_util.get_active_profile_bindings(
self.context)
for name, inp in feed_dict.items():
binding = start_binding + self.context.engine[name]
# Only set shapes if required.
# get_shape/get_binding_shape will return what a shape input/data input is currently set to.
if is_dynamic_shape_input(binding): # For input shape tensors
if isinstance(inp, cuda.DeviceView):
G_LOGGER.critical(
"A DeviceView was provided for input: {:}, but since this is a "
"shape tensor, it must reside in host memory. "
"Please use a NumPy array instead. ".format(name))
if tuple(self.context.get_shape(binding)) != tuple(inp):
G_LOGGER.verbose(
"Setting shape binding: {:} (index: {:}) to: {:}".
format(name, binding, inp))
self.context.set_shape_input(binding, inp)
elif util.is_shape_dynamic(
self.context.engine.get_binding_shape(binding)):
shape = inp.shape
if tuple(self.context.get_binding_shape(binding)) != tuple(
shape):
G_LOGGER.verbose(
"Setting binding: {:} (index: {:}) to shape: {:}".
format(name, binding, shape))
self.context.set_binding_shape(binding, shape)
if not self.context.all_binding_shapes_specified:
G_LOGGER.critical("Some input shapes were not specified.\n"
"Note: Network inputs are: {:}".format(
self.get_input_metadata()))
if not self.context.all_shape_inputs_specified:
G_LOGGER.critical("Some shape inputs were not specified.\n"
"Note: Network inputs are: {:}".format(
self.get_input_metadata()))
return start_binding, end_binding
def get_input_metadata_impl(self):
start_binding, end_binding = trt_util.get_active_profile_bindings(
self.context)
# This function always uses binding names of the 0th profile.
return trt_util.get_input_metadata_from_engine(self.context.engine,
start_binding,
end_binding)
def set_shapes_from_feed_dict(self, feed_dict):
"""
Sets context shapes according to the provided feed_dict, then resizes
buffers as needed.
Args:
feed_dict (OrderedDict[str, numpy.ndarray]): A mapping of input tensor names to corresponding input NumPy arrays.
Returns:
Tuple[int, int]: The start and end binding indices of the modified bindings.
"""
def is_dynamic_shape_input(binding):
try:
self.context.engine.get_profile_shape_input(0, binding)
return True
except RuntimeError:
return False
start_binding, end_binding = trt_util.get_active_profile_bindings(
self.context)
for name, inp in feed_dict.items():
binding = start_binding + self.context.engine[name]
shape = inp.shape
# Only set shapes if required.
# get_shape/get_binding_shape will return what a shape input/data input is currently set to.
if is_dynamic_shape_input(binding): # For input shape tensors
G_LOGGER.verbose(
"Setting shape binding: {:} (index: {:}) to: {:}".format(
name, binding, inp))
if tuple(self.context.get_shape(binding)) != tuple(inp):
self.context.set_shape_input(binding, inp)
elif misc.is_shape_dynamic(
self.context.engine.get_binding_shape(binding)):
G_LOGGER.verbose(
"Setting binding: {:} (index: {:}) to shape: {:}".format(
name, binding, shape))
if tuple(self.context.get_binding_shape(binding)) != tuple(
shape):
self.context.set_binding_shape(binding, shape)
if not self.context.all_binding_shapes_specified:
G_LOGGER.critical(
"Some input shapes were not specified.\nNote: Network inputs are: {:}"
.format(self.get_input_metadata()))
if not self.context.all_shape_inputs_specified:
G_LOGGER.critical(
"Some shape inputs were not specified.\nNote: Network inputs are: {:}"
.format(self.get_input_metadata()))
# Resize device buffers - host buffers will be automatically resized by copy_to
for binding in range(start_binding, end_binding):
name = self.context.engine[
binding -
start_binding] # Use profile 0 binding names for all buffers.
shape = tuple(self.context.get_binding_shape(binding))
self.device_buffers[name].resize(shape)
return start_binding, end_binding
def infer(self, feed_dict):
def is_dynamic_shape_input(binding):
try:
self.engine.get_profile_shape_input(0, binding)
return True
except RuntimeError:
return False
start_binding, end_binding = trt_util.get_active_profile_bindings(
self.engine, self.context)
for name, inp in feed_dict.items():
binding = start_binding + self.engine[name]
shape = inp.shape
# Only set shapes if required.
# get_shape/get_binding_shape will return what a shape input/data input is currently set to.
if is_dynamic_shape_input(binding):
G_LOGGER.verbose(
"Setting shape binding: {:} (index: {:}) to: {:}".format(
name, binding, inp))
if tuple(self.context.get_shape(binding)) != tuple(inp):
self.context.set_shape_input(binding, inp)
elif misc.is_shape_dynamic(self.engine.get_binding_shape(binding)):
G_LOGGER.verbose(
"Setting binding: {:} (index: {:}) to shape: {:}".format(
name, binding, shape))
if tuple(self.context.get_binding_shape(binding)) != tuple(
shape):
self.context.set_binding_shape(binding, shape)
if not self.context.all_binding_shapes_specified:
G_LOGGER.critical(
"Some input shapes were not specified.\nNote: Network inputs are: {:}"
.format(self.get_input_metadata()))
if not self.context.all_shape_inputs_specified:
G_LOGGER.critical(
"Some shape inputs were not specified.\nNote: Network inputs are: {:}"
.format(self.get_input_metadata()))
# Inference
# Need to resize output buffers
self.buffers.resize(self.engine,
self.context,
start_binding=start_binding,
end_binding=end_binding)
start = time.time()
self.buffers.copy_inputs(feed_dict, self.stream)
# Need to offset bindings in case the active profile is not 0.
status = self.context.execute_async_v2(
bindings=[0] * start_binding + self.buffers.bindings(),
stream_handle=self.stream.address())
if not status:
G_LOGGER.critical(
"Model execution failed. Please see the log messages above for details"
)
self.buffers.copy_outputs(self.stream)
self.stream.synchronize()
end = time.time()
self.inference_time = end - start
return self.buffers.outputs