def test_basic(self): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: assert runner.is_active model.check_runner(runner) assert not runner.is_active assert runner._cached_input_metadata is None
def main(): # The OnnxrtRunner requires an ONNX-RT session. # We can use the SessionFromOnnx lazy loader to construct one easily: build_onnxrt_session = SessionFromOnnx("identity.onnx") # The TrtRunner requires a TensorRT engine. # To create one from the ONNX model, we can chain a couple lazy loaders together: build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx")) runners = [ TrtRunner(build_engine), OnnxrtRunner(build_onnxrt_session), ] # `Comparator.run()` will run each runner separately using synthetic input data and # return a `RunResults` instance. See `polygraphy/comparator/struct.py` for details. # # TIP: To use custom input data, you can set the `data_loader` parameter in `Comparator.run()`` # to a generator or iterable that yields `Dict[str, np.ndarray]`. run_results = Comparator.run(runners) # `Comparator.compare_accuracy()` checks that outputs match between runners. # # TIP: The `compare_func` parameter can be used to control how outputs are compared (see API reference for details). # The default comparison function is created by `CompareFunc.simple()`, but we can construct it # explicitly if we want to change the default parameters, such as tolerance. assert bool( Comparator.compare_accuracy( run_results, compare_func=CompareFunc.simple(atol=1e-8))) # We can use `RunResults.save()` method to save the inference results to a JSON file. # This can be useful if you want to generate and compare results separately. run_results.save("inference_results.json")
def test_basic(self): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: assert runner.is_active model.check_runner(runner) assert runner.last_inference_time() is not None assert not runner.is_active
def test_error_on_wrong_shape_feed_dict(self): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: with pytest.raises(PolygraphyException, match="incompatible shape."): runner.infer( {"x": np.ones(shape=(1, 1, 3, 2), dtype=np.float32)})
def test_error_on_wrong_name_feed_dict(self, names, err): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: with pytest.raises(PolygraphyException, match=err): runner.infer({ name: np.ones(shape=(1, 1, 2, 2), dtype=np.float32) for name in names })
def test_dim_param_preserved(self): model = ONNX_MODELS["dim_param"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: input_meta = runner.get_input_metadata() # In Polygraphy, we only use None to indicate a dynamic input dimension - not strings. assert len(input_meta) == 1 for _, (_, shape) in input_meta.items(): assert shape == ['dim0', 16, 128]
def test_postprocess(self): onnx_loader = ONNX_MODELS["identity"].loader run_results = Comparator.run([OnnxrtRunner(SessionFromOnnx(onnx_loader))], use_subprocess=True) # Output shape is (1, 1, 2, 2) postprocessed = Comparator.postprocess(run_results, postprocess_func=PostprocessFunc.topk_func(k=1, axis=-1)) for _, results in postprocessed.items(): for result in results: for _, output in result.items(): assert output.shape == (1, 1, 2, 1)
def test_list_as_data_loader(self): onnx_loader = ONNX_MODELS["identity"].loader runner = OnnxrtRunner(SessionFromOnnx(onnx_loader), name="onnx_runner") data = [{"x": np.ones((1, 1, 2, 2), dtype=np.float32)}] * 2 run_results = Comparator.run([runner], data_loader=data) iter_results = run_results["onnx_runner"] assert len(iter_results) == 2 for actual, expected in zip(iter_results, data): assert np.all(actual["y"] == expected["x"])
def test_dim_param_trt_onnxrt(self): load_onnx_bytes = ONNX_MODELS["dim_param"].loader build_onnxrt_session = SessionFromOnnx(load_onnx_bytes) load_engine = EngineFromNetwork(NetworkFromOnnxBytes(load_onnx_bytes)) runners = [ OnnxrtRunner(build_onnxrt_session), TrtRunner(load_engine), ] run_results = Comparator.run(runners) compare_func = CompareFunc.simple(check_shapes=mod.version(trt.__version__) >= mod.version("7.0")) assert bool(Comparator.compare_accuracy(run_results, compare_func=compare_func)) assert len(list(run_results.values())[0]) == 1 # Default number of iterations
def test_generator_as_data_loader(self): onnx_loader = ONNX_MODELS["identity"].loader runner = OnnxrtRunner(SessionFromOnnx(onnx_loader), name="onnx_runner") def data(): for feed_dict in [{ "x": np.ones((1, 1, 2, 2), dtype=np.float32) }] * 2: yield feed_dict run_results = Comparator.run([runner], data_loader=data()) iter_results = run_results["onnx_runner"] assert len(iter_results) == 2 for actual, expected in zip(iter_results, data()): assert np.all(actual['y'] == expected['x'])
def test_multiple_runners(self): load_tf = TF_MODELS["identity"].loader build_tf_session = SessionFromGraph(load_tf) onnx_model = OnnxFromTfGraph(load_tf) load_serialized_onnx = BytesFromOnnx(onnx_model) build_onnxrt_session = SessionFromOnnx(load_serialized_onnx) load_engine = EngineFromNetwork(NetworkFromOnnxBytes(load_serialized_onnx)) gs_graph = GsFromOnnx(onnx_model) runners = [ TfRunner(build_tf_session), OnnxrtRunner(build_onnxrt_session), PluginRefRunner(gs_graph), TrtRunner(load_engine), ] run_results = Comparator.run(runners) compare_func = CompareFunc.simple(check_shapes=mod.version(trt.__version__) >= mod.version("7.0")) assert bool(Comparator.compare_accuracy(run_results, compare_func=compare_func)) assert len(list(run_results.values())[0]) == 1 # Default number of iterations
def main(): # The OnnxrtRunner requires an ONNX-RT session. # We can use the SessionFromOnnx lazy loader to construct one easily: build_onnxrt_session = SessionFromOnnx("identity.onnx") # The TrtRunner requires a TensorRT engine. # To create one from the ONNX model, we can chain a couple lazy loaders together: build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx")) runners = [ TrtRunner(build_engine), OnnxrtRunner(build_onnxrt_session), ] # `Comparator.run()` will run each runner separately using synthetic input data and return a `RunResults` instance. # See `polygraphy/comparator/struct.py` for details. run_results = Comparator.run(runners) # `Comparator.compare_accuracy()` checks that outputs match between runners. assert bool(Comparator.compare_accuracy(run_results)) # We can use `RunResults.save()` method to save the inference results to a JSON file. # This can be useful if you want to generate and compare results separately. run_results.save("inference_results.json")
from polygraphy.logger import G_LOGGER G_LOGGER.severity = G_LOGGER.VERBOSE from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnx from polygraphy.backend.trt import CreateConfig as CreateTrtConfig, EngineFromNetwork, NetworkFromOnnxPath, Profile, SaveEngine, TrtRunner from polygraphy.common import TensorMetadata from polygraphy.comparator import Comparator, CompareFunc, DataLoader import sys # Data Loader data_loader = DataLoader( input_metadata=TensorMetadata().add('tensor-0', None, (4, 1, 28, 28))) # Loaders build_onnxrt_session = SessionFromOnnx( '/work/gitlab/tensorrt-cookbook-in-chinese/08-Tool/Polygraphy/runExample/model.onnx' ) parse_network_from_onnx = NetworkFromOnnxPath( '/work/gitlab/tensorrt-cookbook-in-chinese/08-Tool/Polygraphy/runExample/model.onnx' ) profiles = [ Profile().add('tensor-0', min=[1, 1, 28, 28], opt=[4, 1, 28, 28], max=[16, 1, 28, 28]) ] create_trt_config = CreateTrtConfig(max_workspace_size=1000000000, profiles=profiles) build_engine = EngineFromNetwork(parse_network_from_onnx, config=create_trt_config) save_engine = SaveEngine(build_engine, path='model-FP32.plan')
def test_infer_raises_if_runner_inactive(): runner = OnnxrtRunner(SessionFromOnnx(ONNX_MODELS["identity"].loader)) feed_dict = {"x": np.ones((1, 1, 2, 2), dtype=np.float32)} with pytest.raises(PolygraphyException, match="Must be activated"): runner.infer(feed_dict)
def test_warmup_runs(self): onnx_loader = ONNX_MODELS["identity"].loader runner = OnnxrtRunner(SessionFromOnnx(onnx_loader)) run_results = Comparator.run([runner], warm_up=2) assert len(run_results[runner.name]) == 1
def test_shape_output(self): model = ONNX_MODELS["reshape"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: model.check_runner(runner)
def __init__(self, onnx_fpath: str, network_metadata: NetworkMetadata): self.network_metadata = network_metadata self.trt_session = SessionFromOnnx(onnx_fpath) self.trt_context = OnnxrtRunner(self.trt_session) self.trt_context.activate()