def test_basic(self): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: assert runner.is_active model.check_runner(runner) assert not runner.is_active assert runner._cached_input_metadata is None
def test_error_on_wrong_shape_feed_dict(self): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: with pytest.raises(PolygraphyException, match="incompatible shape."): runner.infer( {"x": np.ones(shape=(1, 1, 3, 2), dtype=np.float32)})
def test_basic(self): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: assert runner.is_active model.check_runner(runner) assert runner.last_inference_time() is not None assert not runner.is_active
def main(): # The OnnxrtRunner requires an ONNX-RT session. # We can use the SessionFromOnnx lazy loader to construct one easily: build_onnxrt_session = SessionFromOnnx("identity.onnx") # The TrtRunner requires a TensorRT engine. # To create one from the ONNX model, we can chain a couple lazy loaders together: build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx")) runners = [ TrtRunner(build_engine), OnnxrtRunner(build_onnxrt_session), ] # `Comparator.run()` will run each runner separately using synthetic input data and # return a `RunResults` instance. See `polygraphy/comparator/struct.py` for details. # # TIP: To use custom input data, you can set the `data_loader` parameter in `Comparator.run()`` # to a generator or iterable that yields `Dict[str, np.ndarray]`. run_results = Comparator.run(runners) # `Comparator.compare_accuracy()` checks that outputs match between runners. # # TIP: The `compare_func` parameter can be used to control how outputs are compared (see API reference for details). # The default comparison function is created by `CompareFunc.simple()`, but we can construct it # explicitly if we want to change the default parameters, such as tolerance. assert bool( Comparator.compare_accuracy( run_results, compare_func=CompareFunc.simple(atol=1e-8))) # We can use `RunResults.save()` method to save the inference results to a JSON file. # This can be useful if you want to generate and compare results separately. run_results.save("inference_results.json")
class PolygraphyOnnxRunner: def __init__(self, onnx_fpath: str, network_metadata: NetworkMetadata): self.network_metadata = network_metadata self.trt_session = SessionFromOnnx(onnx_fpath) self.trt_context = OnnxrtRunner(self.trt_session) self.trt_context.activate() def __call__(self, *args, **kwargs): # hook polygraphy verbosity for inference g_logger_verbosity = (G_LOGGER.EXTRA_VERBOSE if G_LOGGER.root.level == G_LOGGER.DEBUG else G_LOGGER.WARNING) with PG_LOGGER.verbosity(g_logger_verbosity): return self.forward(*args, **kwargs) def release(self): self.trt_context.deactivate()
def test_dim_param_converted_to_int_shape(self): model = ONNX_MODELS["dim_param"] with OnnxrtRunner(SessionFromOnnxBytes(model.loader)) as runner: input_meta = runner.get_input_metadata() # In Polygraphy, we only use None to indicate a dynamic input dimension - not strings. for name, (dtype, shape) in input_meta.items(): for dim in shape: assert dim is None or isinstance(dim, int)
def test_error_on_wrong_name_feed_dict(self, names, err): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: with pytest.raises(PolygraphyException, match=err): runner.infer({ name: np.ones(shape=(1, 1, 2, 2), dtype=np.float32) for name in names })
def test_dim_param_preserved(self): model = ONNX_MODELS["dim_param"] with OnnxrtRunner(SessionFromOnnx(model.loader)) as runner: input_meta = runner.get_input_metadata() # In Polygraphy, we only use None to indicate a dynamic input dimension - not strings. assert len(input_meta) == 1 for _, (_, shape) in input_meta.items(): assert shape == ['dim0', 16, 128]
def test_postprocess(self): onnx_loader = ONNX_MODELS["identity"].loader run_results = Comparator.run([OnnxrtRunner(SessionFromOnnx(onnx_loader))], use_subprocess=True) # Output shape is (1, 1, 2, 2) postprocessed = Comparator.postprocess(run_results, postprocess_func=PostprocessFunc.topk_func(k=1, axis=-1)) for _, results in postprocessed.items(): for result in results: for _, output in result.items(): assert output.shape == (1, 1, 2, 1)
def test_multirun_outputs_are_different(self): onnx_loader = ONNX_MODELS["identity"].loader runner = OnnxrtRunner(SessionFromOnnxBytes(onnx_loader)) run_results = Comparator.run([runner], data_loader=DataLoader(iterations=2)) iteration0 = run_results[runner.name][0] iteration1 = run_results[runner.name][1] for name in iteration0.keys(): assert np.any(iteration0[name] != iteration1[name])
def test_list_as_data_loader(self): onnx_loader = ONNX_MODELS["identity"].loader runner = OnnxrtRunner(SessionFromOnnx(onnx_loader), name="onnx_runner") data = [{"x": np.ones((1, 1, 2, 2), dtype=np.float32)}] * 2 run_results = Comparator.run([runner], data_loader=data) iter_results = run_results["onnx_runner"] assert len(iter_results) == 2 for actual, expected in zip(iter_results, data): assert np.all(actual["y"] == expected["x"])
def test_generator_as_data_loader(self): onnx_loader = ONNX_MODELS["identity"].loader runner = OnnxrtRunner(SessionFromOnnxBytes(onnx_loader), name="onnx_runner") def data(): for feed_dict in [{"x": np.ones((1, 1, 2, 2), dtype=np.float32)}] * 2: yield feed_dict run_results = Comparator.run([runner], data_loader=data()) iter_results = run_results["onnx_runner"] assert len(iter_results) == 2 for actual, expected in zip(iter_results, data()): assert np.all(actual['y'] == expected['x'])
def test_dim_param_trt_onnxrt(self): load_onnx_bytes = ONNX_MODELS["dim_param"].loader build_onnxrt_session = SessionFromOnnx(load_onnx_bytes) load_engine = EngineFromNetwork(NetworkFromOnnxBytes(load_onnx_bytes)) runners = [ OnnxrtRunner(build_onnxrt_session), TrtRunner(load_engine), ] run_results = Comparator.run(runners) compare_func = CompareFunc.simple(check_shapes=mod.version(trt.__version__) >= mod.version("7.0")) assert bool(Comparator.compare_accuracy(run_results, compare_func=compare_func)) assert len(list(run_results.values())[0]) == 1 # Default number of iterations
def fallback_shape_inference(onnx_model): from polygraphy.backend.onnx import BytesFromOnnx, ModifyOnnx from polygraphy.backend.onnxrt import (OnnxrtRunner, SessionFromOnnxBytes) load_model = ModifyOnnx(onnx_model, outputs=constants.MARK_ALL) with OnnxrtRunner(SessionFromOnnxBytes(BytesFromOnnx(load_model))) as runner: data_loader = self.makers[DataLoaderArgs].get_data_loader() data_loader.input_metadata = runner.get_input_metadata() outputs = runner.infer(feed_dict=data_loader[0]) meta = TensorMetadata() for name, output in outputs.items(): meta.add(name, output.dtype, output.shape) return meta
def test_multiple_runners(self): load_tf = TF_MODELS["identity"].loader build_tf_session = SessionFromGraph(load_tf) load_serialized_onnx = BytesFromOnnx(OnnxFromTfGraph(load_tf)) build_onnxrt_session = SessionFromOnnxBytes(load_serialized_onnx) load_engine = EngineFromNetwork(NetworkFromOnnxBytes(load_serialized_onnx)) runners = [ TfRunner(build_tf_session), OnnxrtRunner(build_onnxrt_session), TrtRunner(load_engine), ] run_results = Comparator.run(runners) compare_func = CompareFunc.basic_compare_func(check_shapes=version(trt.__version__) >= version("7.0")) assert bool(Comparator.compare_accuracy(run_results, compare_func=compare_func)) assert len(list(run_results.values())[0]) == 1 # Default number of iterations
def main(): # The OnnxrtRunner requires an ONNX-RT session. # We can use the SessionFromOnnx lazy loader to construct one easily: build_onnxrt_session = SessionFromOnnx("identity.onnx") # The TrtRunner requires a TensorRT engine. # To create one from the ONNX model, we can chain a couple lazy loaders together: build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx")) runners = [ TrtRunner(build_engine), OnnxrtRunner(build_onnxrt_session), ] # `Comparator.run()` will run each runner separately using synthetic input data and return a `RunResults` instance. # See `polygraphy/comparator/struct.py` for details. run_results = Comparator.run(runners) # `Comparator.compare_accuracy()` checks that outputs match between runners. assert bool(Comparator.compare_accuracy(run_results)) # We can use `RunResults.save()` method to save the inference results to a JSON file. # This can be useful if you want to generate and compare results separately. run_results.save("inference_results.json")
def test_basic(self): model = ONNX_MODELS["identity"] with OnnxrtRunner(SessionFromOnnxBytes(model.loader)) as runner: assert runner.is_active model.check_runner(runner) assert not runner.is_active
def test_infer_raises_if_runner_inactive(): runner = OnnxrtRunner(SessionFromOnnx(ONNX_MODELS["identity"].loader)) feed_dict = {"x": np.ones((1, 1, 2, 2), dtype=np.float32)} with pytest.raises(PolygraphyException, match="Must be activated"): runner.infer(feed_dict)
# limitations under the License. # """ This script runs an identity model with ONNX-Runtime and TensorRT, then compares outputs. """ from polygraphy.backend.trt import NetworkFromOnnxBytes, EngineFromNetwork, TrtRunner from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnxBytes from polygraphy.backend.common import BytesFromPath from polygraphy.comparator import Comparator import os # Create loaders for both ONNX Runtime and TensorRT MODEL = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, "models", "identity.onnx") load_serialized_onnx = BytesFromPath(MODEL) build_onnxrt_session = SessionFromOnnxBytes(load_serialized_onnx) build_engine = EngineFromNetwork(NetworkFromOnnxBytes(load_serialized_onnx)) # Create runners runners = [ TrtRunner(build_engine), OnnxrtRunner(build_onnxrt_session), ] # Finally, run and compare the results. run_results = Comparator.run(runners) assert bool(Comparator.compare_accuracy(run_results))
def test_can_name_runner(self): NAME = "runner" runner = OnnxrtRunner(None, name=NAME) assert runner.name == NAME
def __init__(self, onnx_fpath: str, network_metadata: NetworkMetadata): self.network_metadata = network_metadata self.trt_session = SessionFromOnnx(onnx_fpath) self.trt_context = OnnxrtRunner(self.trt_session) self.trt_context.activate()
def test_warmup_runs(self): onnx_loader = ONNX_MODELS["identity"].loader runner = OnnxrtRunner(SessionFromOnnxBytes(onnx_loader)) run_results = Comparator.run([runner], warm_up=2) assert len(run_results[runner.name]) == 1
def test_shape_output(self): model = ONNX_MODELS["reshape"] with OnnxrtRunner(SessionFromOnnxBytes(model.loader)) as runner: model.check_runner(runner)