def test_basic(self):
model = ONNX_MODELS["identity"]
with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner:
assert runner.is_active
model.check_runner(runner)
assert not runner.is_active
assert runner._cached_input_metadata is None
def main():
# The OnnxrtRunner requires an ONNX-RT session.
# We can use the SessionFromOnnx lazy loader to construct one easily:
build_onnxrt_session = SessionFromOnnx("identity.onnx")
# The TrtRunner requires a TensorRT engine.
# To create one from the ONNX model, we can chain a couple lazy loaders together:
build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx"))
runners = [
TrtRunner(build_engine),
OnnxrtRunner(build_onnxrt_session),
]
# `Comparator.run()` will run each runner separately using synthetic input data and
# return a `RunResults` instance. See `polygraphy/comparator/struct.py` for details.
#
# TIP: To use custom input data, you can set the `data_loader` parameter in `Comparator.run()``
# to a generator or iterable that yields `Dict[str, np.ndarray]`.
run_results = Comparator.run(runners)
# `Comparator.compare_accuracy()` checks that outputs match between runners.
#
# TIP: The `compare_func` parameter can be used to control how outputs are compared (see API reference for details).
# The default comparison function is created by `CompareFunc.simple()`, but we can construct it
# explicitly if we want to change the default parameters, such as tolerance.
assert bool(
Comparator.compare_accuracy(
run_results, compare_func=CompareFunc.simple(atol=1e-8)))
# We can use `RunResults.save()` method to save the inference results to a JSON file.
# This can be useful if you want to generate and compare results separately.
run_results.save("inference_results.json")
def test_basic(self):
model = ONNX_MODELS["identity"]
with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner:
assert runner.is_active
model.check_runner(runner)
assert runner.last_inference_time() is not None
assert not runner.is_active
def test_error_on_wrong_shape_feed_dict(self):
model = ONNX_MODELS["identity"]
with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner:
with pytest.raises(PolygraphyException,
match="incompatible shape."):
runner.infer(
{"x": np.ones(shape=(1, 1, 3, 2), dtype=np.float32)})
def test_error_on_wrong_name_feed_dict(self, names, err):
model = ONNX_MODELS["identity"]
with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner:
with pytest.raises(PolygraphyException, match=err):
runner.infer({
name: np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
for name in names
})
def test_dim_param_preserved(self):
model = ONNX_MODELS["dim_param"]
with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner:
input_meta = runner.get_input_metadata()
# In Polygraphy, we only use None to indicate a dynamic input dimension - not strings.
assert len(input_meta) == 1
for _, (_, shape) in input_meta.items():
assert shape == ['dim0', 16, 128]
def test_postprocess(self):
onnx_loader = ONNX_MODELS["identity"].loader
run_results = Comparator.run([OnnxrtRunner(SessionFromOnnx(onnx_loader))], use_subprocess=True)
# Output shape is (1, 1, 2, 2)
postprocessed = Comparator.postprocess(run_results, postprocess_func=PostprocessFunc.topk_func(k=1, axis=-1))
for _, results in postprocessed.items():
for result in results:
for _, output in result.items():
assert output.shape == (1, 1, 2, 1)
def test_list_as_data_loader(self):
onnx_loader = ONNX_MODELS["identity"].loader
runner = OnnxrtRunner(SessionFromOnnx(onnx_loader), name="onnx_runner")
data = [{"x": np.ones((1, 1, 2, 2), dtype=np.float32)}] * 2
run_results = Comparator.run([runner], data_loader=data)
iter_results = run_results["onnx_runner"]
assert len(iter_results) == 2
for actual, expected in zip(iter_results, data):
assert np.all(actual["y"] == expected["x"])
def test_dim_param_trt_onnxrt(self):
load_onnx_bytes = ONNX_MODELS["dim_param"].loader
build_onnxrt_session = SessionFromOnnx(load_onnx_bytes)
load_engine = EngineFromNetwork(NetworkFromOnnxBytes(load_onnx_bytes))
runners = [
OnnxrtRunner(build_onnxrt_session),
TrtRunner(load_engine),
]
run_results = Comparator.run(runners)
compare_func = CompareFunc.simple(check_shapes=mod.version(trt.__version__) >= mod.version("7.0"))
assert bool(Comparator.compare_accuracy(run_results, compare_func=compare_func))
assert len(list(run_results.values())[0]) == 1 # Default number of iterations
def test_generator_as_data_loader(self):
onnx_loader = ONNX_MODELS["identity"].loader
runner = OnnxrtRunner(SessionFromOnnx(onnx_loader), name="onnx_runner")
def data():
for feed_dict in [{
"x": np.ones((1, 1, 2, 2), dtype=np.float32)
}] * 2:
yield feed_dict
run_results = Comparator.run([runner], data_loader=data())
iter_results = run_results["onnx_runner"]
assert len(iter_results) == 2
for actual, expected in zip(iter_results, data()):
assert np.all(actual['y'] == expected['x'])
def test_multiple_runners(self):
load_tf = TF_MODELS["identity"].loader
build_tf_session = SessionFromGraph(load_tf)
onnx_model = OnnxFromTfGraph(load_tf)
load_serialized_onnx = BytesFromOnnx(onnx_model)
build_onnxrt_session = SessionFromOnnx(load_serialized_onnx)
load_engine = EngineFromNetwork(NetworkFromOnnxBytes(load_serialized_onnx))
gs_graph = GsFromOnnx(onnx_model)
runners = [
TfRunner(build_tf_session),
OnnxrtRunner(build_onnxrt_session),
PluginRefRunner(gs_graph),
TrtRunner(load_engine),
]
run_results = Comparator.run(runners)
compare_func = CompareFunc.simple(check_shapes=mod.version(trt.__version__) >= mod.version("7.0"))
assert bool(Comparator.compare_accuracy(run_results, compare_func=compare_func))
assert len(list(run_results.values())[0]) == 1 # Default number of iterations
def main():
# The OnnxrtRunner requires an ONNX-RT session.
# We can use the SessionFromOnnx lazy loader to construct one easily:
build_onnxrt_session = SessionFromOnnx("identity.onnx")
# The TrtRunner requires a TensorRT engine.
# To create one from the ONNX model, we can chain a couple lazy loaders together:
build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx"))
runners = [
TrtRunner(build_engine),
OnnxrtRunner(build_onnxrt_session),
]
# `Comparator.run()` will run each runner separately using synthetic input data and return a `RunResults` instance.
# See `polygraphy/comparator/struct.py` for details.
run_results = Comparator.run(runners)
# `Comparator.compare_accuracy()` checks that outputs match between runners.
assert bool(Comparator.compare_accuracy(run_results))
# We can use `RunResults.save()` method to save the inference results to a JSON file.
# This can be useful if you want to generate and compare results separately.
run_results.save("inference_results.json")
from polygraphy.logger import G_LOGGER
G_LOGGER.severity = G_LOGGER.VERBOSE
from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnx
from polygraphy.backend.trt import CreateConfig as CreateTrtConfig, EngineFromNetwork, NetworkFromOnnxPath, Profile, SaveEngine, TrtRunner
from polygraphy.common import TensorMetadata
from polygraphy.comparator import Comparator, CompareFunc, DataLoader
import sys
# Data Loader
data_loader = DataLoader(
input_metadata=TensorMetadata().add('tensor-0', None, (4, 1, 28, 28)))
# Loaders
build_onnxrt_session = SessionFromOnnx(
'/work/gitlab/tensorrt-cookbook-in-chinese/08-Tool/Polygraphy/runExample/model.onnx'
)
parse_network_from_onnx = NetworkFromOnnxPath(
'/work/gitlab/tensorrt-cookbook-in-chinese/08-Tool/Polygraphy/runExample/model.onnx'
)
profiles = [
Profile().add('tensor-0',
min=[1, 1, 28, 28],
opt=[4, 1, 28, 28],
max=[16, 1, 28, 28])
]
create_trt_config = CreateTrtConfig(max_workspace_size=1000000000,
profiles=profiles)
build_engine = EngineFromNetwork(parse_network_from_onnx,
config=create_trt_config)
save_engine = SaveEngine(build_engine, path='model-FP32.plan')
def test_infer_raises_if_runner_inactive():
runner = OnnxrtRunner(SessionFromOnnx(ONNX_MODELS["identity"].loader))
feed_dict = {"x": np.ones((1, 1, 2, 2), dtype=np.float32)}
with pytest.raises(PolygraphyException, match="Must be activated"):
runner.infer(feed_dict)
def test_warmup_runs(self):
onnx_loader = ONNX_MODELS["identity"].loader
runner = OnnxrtRunner(SessionFromOnnx(onnx_loader))
run_results = Comparator.run([runner], warm_up=2)
assert len(run_results[runner.name]) == 1
def test_shape_output(self):
model = ONNX_MODELS["reshape"]
with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner:
model.check_runner(runner)
def __init__(self, onnx_fpath: str, network_metadata: NetworkMetadata):
self.network_metadata = network_metadata
self.trt_session = SessionFromOnnx(onnx_fpath)
self.trt_context = OnnxrtRunner(self.trt_session)
self.trt_context.activate()
