def _create_inference_session(self, providers, provider_options):
# Tensorrt can fall back to CUDA. All others fall back to CPU.
if 'TensorrtExecutionProvider' in C.get_available_providers():
self._fallback_providers = [
'CUDAExecutionProvider', 'CPUExecutionProvider'
]
else:
self._fallback_providers = ['CPUExecutionProvider']
session_options = self._sess_options if self._sess_options else C.get_default_session_options(
)
if self._model_path:
sess = C.InferenceSession(session_options, self._model_path, True,
self._read_config_from_model)
else:
sess = C.InferenceSession(session_options, self._model_bytes,
False, self._read_config_from_model)
# initialize the C++ InferenceSession
sess.initialize_session(providers or [], provider_options or [])
self._sess = sess
self._sess_options = self._sess.session_options
self._inputs_meta = self._sess.inputs_meta
self._outputs_meta = self._sess.outputs_meta
self._overridable_initializers = self._sess.overridable_initializers
self._model_meta = self._sess.model_meta
self._providers = self._sess.get_providers()
self._provider_options = self._sess.get_provider_options()
def _load_model(self, providers):
if isinstance(self._path_or_bytes, str):
self._sess = C.InferenceSession(
self._sess_options if self._sess_options else
C.get_default_session_options(), self._path_or_bytes, True)
elif isinstance(self._path_or_bytes, bytes):
self._sess = C.InferenceSession(
self._sess_options if self._sess_options else
C.get_default_session_options(), self._path_or_bytes, False)
# elif isinstance(self._path_or_bytes, tuple):
# to remove, hidden trick
# self._sess.load_model_no_init(self._path_or_bytes[0], providers)
else:
raise TypeError("Unable to load from type '{0}'".format(
type(self._path_or_bytes)))
self._sess.load_model(providers)
self._sess_options = self._sess.session_options
self._inputs_meta = self._sess.inputs_meta
self._outputs_meta = self._sess.outputs_meta
self._overridable_initializers = self._sess.overridable_initializers
self._model_meta = self._sess.model_meta
self._providers = self._sess.get_providers()
# Tensorrt can fall back to CUDA. All others fall back to CPU.
if 'TensorrtExecutionProvider' in C.get_available_providers():
self._fallback_providers = [
'CUDAExecutionProvider', 'CPUExecutionProvider'
]
else:
self._fallback_providers = ['CPUExecutionProvider']
def __init__(self, path_or_bytes, sess_options=None):
"""
:param path_or_bytes: filename or serialized model in a byte string
:param sess_options: session options
"""
if sess_options:
self._sess = C.InferenceSession(sess_options,
C.get_session_initializer())
else:
self._sess = C.InferenceSession(C.get_session_initializer(),
C.get_session_initializer())
if isinstance(path_or_bytes, str):
self._sess.load_model(path_or_bytes)
elif isinstance(path_or_bytes, bytes):
self._sess.read_bytes(path_or_bytes)
elif isinstance(path_or_bytes, tuple):
# to remove, hidden trick
self._sess.load_model_no_init(path_or_bytes[0])
else:
raise TypeError("Unable to load from type '{0}'".format(
type(path_or_bytes)))
self._inputs_meta = self._sess.inputs_meta
self._outputs_meta = self._sess.outputs_meta
self._model_meta = self._sess.model_meta
def _create_inference_session(self,
providers,
provider_options,
disabled_optimizers=None):
available_providers = C.get_available_providers()
# Tensorrt can fall back to CUDA. All others fall back to CPU.
if 'TensorrtExecutionProvider' in available_providers:
self._fallback_providers = [
'CUDAExecutionProvider', 'CPUExecutionProvider'
]
elif 'MIGraphXExecutionProvider' in available_providers:
self._fallback_providers = [
'ROCMExecutionProvider', 'CPUExecutionProvider'
]
else:
self._fallback_providers = ['CPUExecutionProvider']
# validate providers and provider_options before other initialization
providers, provider_options = check_and_normalize_provider_args(
providers, provider_options, available_providers)
if providers == [] and len(available_providers) > 1:
self.disable_fallback()
raise ValueError(
"This ORT build has {} enabled. ".format(available_providers) +
"Since ORT 1.9, you are required to explicitly set " +
"the providers parameter when instantiating InferenceSession. For example, "
"onnxruntime.InferenceSession(..., providers={}, ...)".format(
available_providers))
session_options = self._sess_options if self._sess_options else C.get_default_session_options(
)
if self._model_path:
sess = C.InferenceSession(session_options, self._model_path, True,
self._read_config_from_model)
else:
sess = C.InferenceSession(session_options, self._model_bytes,
False, self._read_config_from_model)
if disabled_optimizers is None:
disabled_optimizers = set()
elif not isinstance(disabled_optimizers, set):
# convert to set. assumes iterable
disabled_optimizers = set(disabled_optimizers)
# initialize the C++ InferenceSession
sess.initialize_session(providers, provider_options,
disabled_optimizers)
self._sess = sess
self._sess_options = self._sess.session_options
self._inputs_meta = self._sess.inputs_meta
self._outputs_meta = self._sess.outputs_meta
self._overridable_initializers = self._sess.overridable_initializers
self._model_meta = self._sess.model_meta
self._providers = self._sess.get_providers()
self._provider_options = self._sess.get_provider_options()
self._profiling_start_time_ns = self._sess.get_profiling_start_time_ns
def _create_inference_session(self,
providers,
provider_options,
disabled_optimizers=None):
available_providers = C.get_available_providers()
# Tensorrt can fall back to CUDA. All others fall back to CPU.
if 'TensorrtExecutionProvider' in available_providers:
self._fallback_providers = [
'CUDAExecutionProvider', 'CPUExecutionProvider'
]
else:
self._fallback_providers = ['CPUExecutionProvider']
# validate providers and provider_options before other initialization
providers, provider_options = check_and_normalize_provider_args(
providers, provider_options, available_providers)
if providers == [] and len(available_providers) > 1:
warnings.warn(
"Deprecation warning. This ORT build has {} enabled. ".format(
available_providers) +
"The next release (ORT 1.10) will require explicitly setting the providers parameter "
+
"(as opposed to the current behavior of providers getting set/registered by default "
+
"based on the build flags) when instantiating InferenceSession."
"For example, onnxruntime.InferenceSession(..., providers=[\"CUDAExecutionProvider\"], ...)"
)
session_options = self._sess_options if self._sess_options else C.get_default_session_options(
)
if self._model_path:
sess = C.InferenceSession(session_options, self._model_path, True,
self._read_config_from_model)
else:
sess = C.InferenceSession(session_options, self._model_bytes,
False, self._read_config_from_model)
if disabled_optimizers is None:
disabled_optimizers = set()
elif not isinstance(disabled_optimizers, set):
# convert to set. assumes iterable
disabled_optimizers = set(disabled_optimizers)
# initialize the C++ InferenceSession
sess.initialize_session(providers, provider_options,
disabled_optimizers)
self._sess = sess
self._sess_options = self._sess.session_options
self._inputs_meta = self._sess.inputs_meta
self._outputs_meta = self._sess.outputs_meta
self._overridable_initializers = self._sess.overridable_initializers
self._model_meta = self._sess.model_meta
self._providers = self._sess.get_providers()
self._provider_options = self._sess.get_provider_options()
self._profiling_start_time_ns = self._sess.get_profiling_start_time_ns
def testRegisterCustomEPsLibrary(self):
# exclude for macos and linux
if not sys.platform.startswith("win"):
return
# Exclude for training
training_enabled = False
try:
from onnxruntime.capi.ort_trainer import ORTTrainer
training_enabled = True
except:
pass
if training_enabled:
return
shared_library = 'test_execution_provider.dll'
if not os.path.exists(shared_library):
raise FileNotFoundError("Unable to find '{0}'".format(shared_library))
this = os.path.dirname(__file__)
custom_op_model = os.path.join(this, "testdata", "custom_execution_provider_library", "test_model.onnx")
if not os.path.exists(custom_op_model):
raise FileNotFoundError("Unable to find '{0}'".format(custom_op_model))
from onnxruntime.capi import _pybind_state as C
session_options = C.get_default_session_options()
sess = C.InferenceSession(session_options, custom_op_model, True, True)
sess.initialize_session(['my_ep'],
[{'shared_lib_path': shared_library,
'device_id':'1', 'some_config':'val'}],
set())
print("Create session with customize execution provider successfully!")
def _create_inference_session(self,
providers,
provider_options,
disabled_optimizers=None):
available_providers = C.get_available_providers()
# Tensorrt can fall back to CUDA. All others fall back to CPU.
if 'TensorrtExecutionProvider' in available_providers:
self._fallback_providers = [
'CUDAExecutionProvider', 'CPUExecutionProvider'
]
else:
self._fallback_providers = ['CPUExecutionProvider']
# validate providers and provider_options before other initialization
providers, provider_options = check_and_normalize_provider_args(
providers, provider_options, available_providers)
session_options = self._sess_options if self._sess_options else C.get_default_session_options(
)
if self._model_path:
sess = C.InferenceSession(session_options, self._model_path, True,
self._read_config_from_model)
else:
sess = C.InferenceSession(session_options, self._model_bytes,
False, self._read_config_from_model)
if disabled_optimizers is None:
disabled_optimizers = set()
elif not isinstance(disabled_optimizers, set):
# convert to set. assumes iterable
disabled_optimizers = set(disabled_optimizers)
# initialize the C++ InferenceSession
sess.initialize_session(providers, provider_options,
disabled_optimizers)
self._sess = sess
self._sess_options = self._sess.session_options
self._inputs_meta = self._sess.inputs_meta
self._outputs_meta = self._sess.outputs_meta
self._overridable_initializers = self._sess.overridable_initializers
self._model_meta = self._sess.model_meta
self._providers = self._sess.get_providers()
self._provider_options = self._sess.get_provider_options()
self._profiling_start_time_ns = self._sess.get_profiling_start_time_ns
def _load_model(self, providers=[]):
if self._sess_options:
self._sess = C.InferenceSession(
self._sess_options, C.get_session_initializer())
else:
self._sess = C.InferenceSession(
C.get_session_initializer(), C.get_session_initializer())
if isinstance(self._path_or_bytes, str):
self._sess.load_model(self._path_or_bytes, providers)
elif isinstance(self._path_or_bytes, bytes):
self._sess.read_bytes(self._path_or_bytes, providers)
elif isinstance(self._path_or_bytes, tuple):
# to remove, hidden trick
self._sess.load_model_no_init(self._path_or_bytes[0], providers)
else:
raise TypeError("Unable to load from type '{0}'".format(type(self._path_or_bytes)))
self._inputs_meta = self._sess.inputs_meta
self._outputs_meta = self._sess.outputs_meta
self._model_meta = self._sess.model_meta
self._providers = self._sess.get_providers()
