def _create_inference_session(self, providers, provider_options): # Tensorrt can fall back to CUDA. All others fall back to CPU. if 'TensorrtExecutionProvider' in C.get_available_providers(): self._fallback_providers = [ 'CUDAExecutionProvider', 'CPUExecutionProvider' ] else: self._fallback_providers = ['CPUExecutionProvider'] session_options = self._sess_options if self._sess_options else C.get_default_session_options( ) if self._model_path: sess = C.InferenceSession(session_options, self._model_path, True, self._read_config_from_model) else: sess = C.InferenceSession(session_options, self._model_bytes, False, self._read_config_from_model) # initialize the C++ InferenceSession sess.initialize_session(providers or [], provider_options or []) self._sess = sess self._sess_options = self._sess.session_options self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta self._overridable_initializers = self._sess.overridable_initializers self._model_meta = self._sess.model_meta self._providers = self._sess.get_providers() self._provider_options = self._sess.get_provider_options()
def _load_model(self, providers): if isinstance(self._path_or_bytes, str): self._sess = C.InferenceSession( self._sess_options if self._sess_options else C.get_default_session_options(), self._path_or_bytes, True) elif isinstance(self._path_or_bytes, bytes): self._sess = C.InferenceSession( self._sess_options if self._sess_options else C.get_default_session_options(), self._path_or_bytes, False) # elif isinstance(self._path_or_bytes, tuple): # to remove, hidden trick # self._sess.load_model_no_init(self._path_or_bytes[0], providers) else: raise TypeError("Unable to load from type '{0}'".format( type(self._path_or_bytes))) self._sess.load_model(providers) self._sess_options = self._sess.session_options self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta self._overridable_initializers = self._sess.overridable_initializers self._model_meta = self._sess.model_meta self._providers = self._sess.get_providers() # Tensorrt can fall back to CUDA. All others fall back to CPU. if 'TensorrtExecutionProvider' in C.get_available_providers(): self._fallback_providers = [ 'CUDAExecutionProvider', 'CPUExecutionProvider' ] else: self._fallback_providers = ['CPUExecutionProvider']
def __init__(self, path_or_bytes, sess_options=None): """ :param path_or_bytes: filename or serialized model in a byte string :param sess_options: session options """ if sess_options: self._sess = C.InferenceSession(sess_options, C.get_session_initializer()) else: self._sess = C.InferenceSession(C.get_session_initializer(), C.get_session_initializer()) if isinstance(path_or_bytes, str): self._sess.load_model(path_or_bytes) elif isinstance(path_or_bytes, bytes): self._sess.read_bytes(path_or_bytes) elif isinstance(path_or_bytes, tuple): # to remove, hidden trick self._sess.load_model_no_init(path_or_bytes[0]) else: raise TypeError("Unable to load from type '{0}'".format( type(path_or_bytes))) self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta self._model_meta = self._sess.model_meta
def _create_inference_session(self, providers, provider_options, disabled_optimizers=None): available_providers = C.get_available_providers() # Tensorrt can fall back to CUDA. All others fall back to CPU. if 'TensorrtExecutionProvider' in available_providers: self._fallback_providers = [ 'CUDAExecutionProvider', 'CPUExecutionProvider' ] elif 'MIGraphXExecutionProvider' in available_providers: self._fallback_providers = [ 'ROCMExecutionProvider', 'CPUExecutionProvider' ] else: self._fallback_providers = ['CPUExecutionProvider'] # validate providers and provider_options before other initialization providers, provider_options = check_and_normalize_provider_args( providers, provider_options, available_providers) if providers == [] and len(available_providers) > 1: self.disable_fallback() raise ValueError( "This ORT build has {} enabled. ".format(available_providers) + "Since ORT 1.9, you are required to explicitly set " + "the providers parameter when instantiating InferenceSession. For example, " "onnxruntime.InferenceSession(..., providers={}, ...)".format( available_providers)) session_options = self._sess_options if self._sess_options else C.get_default_session_options( ) if self._model_path: sess = C.InferenceSession(session_options, self._model_path, True, self._read_config_from_model) else: sess = C.InferenceSession(session_options, self._model_bytes, False, self._read_config_from_model) if disabled_optimizers is None: disabled_optimizers = set() elif not isinstance(disabled_optimizers, set): # convert to set. assumes iterable disabled_optimizers = set(disabled_optimizers) # initialize the C++ InferenceSession sess.initialize_session(providers, provider_options, disabled_optimizers) self._sess = sess self._sess_options = self._sess.session_options self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta self._overridable_initializers = self._sess.overridable_initializers self._model_meta = self._sess.model_meta self._providers = self._sess.get_providers() self._provider_options = self._sess.get_provider_options() self._profiling_start_time_ns = self._sess.get_profiling_start_time_ns
def _create_inference_session(self, providers, provider_options, disabled_optimizers=None): available_providers = C.get_available_providers() # Tensorrt can fall back to CUDA. All others fall back to CPU. if 'TensorrtExecutionProvider' in available_providers: self._fallback_providers = [ 'CUDAExecutionProvider', 'CPUExecutionProvider' ] else: self._fallback_providers = ['CPUExecutionProvider'] # validate providers and provider_options before other initialization providers, provider_options = check_and_normalize_provider_args( providers, provider_options, available_providers) if providers == [] and len(available_providers) > 1: warnings.warn( "Deprecation warning. This ORT build has {} enabled. ".format( available_providers) + "The next release (ORT 1.10) will require explicitly setting the providers parameter " + "(as opposed to the current behavior of providers getting set/registered by default " + "based on the build flags) when instantiating InferenceSession." "For example, onnxruntime.InferenceSession(..., providers=[\"CUDAExecutionProvider\"], ...)" ) session_options = self._sess_options if self._sess_options else C.get_default_session_options( ) if self._model_path: sess = C.InferenceSession(session_options, self._model_path, True, self._read_config_from_model) else: sess = C.InferenceSession(session_options, self._model_bytes, False, self._read_config_from_model) if disabled_optimizers is None: disabled_optimizers = set() elif not isinstance(disabled_optimizers, set): # convert to set. assumes iterable disabled_optimizers = set(disabled_optimizers) # initialize the C++ InferenceSession sess.initialize_session(providers, provider_options, disabled_optimizers) self._sess = sess self._sess_options = self._sess.session_options self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta self._overridable_initializers = self._sess.overridable_initializers self._model_meta = self._sess.model_meta self._providers = self._sess.get_providers() self._provider_options = self._sess.get_provider_options() self._profiling_start_time_ns = self._sess.get_profiling_start_time_ns
def testRegisterCustomEPsLibrary(self): # exclude for macos and linux if not sys.platform.startswith("win"): return # Exclude for training training_enabled = False try: from onnxruntime.capi.ort_trainer import ORTTrainer training_enabled = True except: pass if training_enabled: return shared_library = 'test_execution_provider.dll' if not os.path.exists(shared_library): raise FileNotFoundError("Unable to find '{0}'".format(shared_library)) this = os.path.dirname(__file__) custom_op_model = os.path.join(this, "testdata", "custom_execution_provider_library", "test_model.onnx") if not os.path.exists(custom_op_model): raise FileNotFoundError("Unable to find '{0}'".format(custom_op_model)) from onnxruntime.capi import _pybind_state as C session_options = C.get_default_session_options() sess = C.InferenceSession(session_options, custom_op_model, True, True) sess.initialize_session(['my_ep'], [{'shared_lib_path': shared_library, 'device_id':'1', 'some_config':'val'}], set()) print("Create session with customize execution provider successfully!")
def _create_inference_session(self, providers, provider_options, disabled_optimizers=None): available_providers = C.get_available_providers() # Tensorrt can fall back to CUDA. All others fall back to CPU. if 'TensorrtExecutionProvider' in available_providers: self._fallback_providers = [ 'CUDAExecutionProvider', 'CPUExecutionProvider' ] else: self._fallback_providers = ['CPUExecutionProvider'] # validate providers and provider_options before other initialization providers, provider_options = check_and_normalize_provider_args( providers, provider_options, available_providers) session_options = self._sess_options if self._sess_options else C.get_default_session_options( ) if self._model_path: sess = C.InferenceSession(session_options, self._model_path, True, self._read_config_from_model) else: sess = C.InferenceSession(session_options, self._model_bytes, False, self._read_config_from_model) if disabled_optimizers is None: disabled_optimizers = set() elif not isinstance(disabled_optimizers, set): # convert to set. assumes iterable disabled_optimizers = set(disabled_optimizers) # initialize the C++ InferenceSession sess.initialize_session(providers, provider_options, disabled_optimizers) self._sess = sess self._sess_options = self._sess.session_options self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta self._overridable_initializers = self._sess.overridable_initializers self._model_meta = self._sess.model_meta self._providers = self._sess.get_providers() self._provider_options = self._sess.get_provider_options() self._profiling_start_time_ns = self._sess.get_profiling_start_time_ns
def _load_model(self, providers=[]): if self._sess_options: self._sess = C.InferenceSession( self._sess_options, C.get_session_initializer()) else: self._sess = C.InferenceSession( C.get_session_initializer(), C.get_session_initializer()) if isinstance(self._path_or_bytes, str): self._sess.load_model(self._path_or_bytes, providers) elif isinstance(self._path_or_bytes, bytes): self._sess.read_bytes(self._path_or_bytes, providers) elif isinstance(self._path_or_bytes, tuple): # to remove, hidden trick self._sess.load_model_no_init(self._path_or_bytes[0], providers) else: raise TypeError("Unable to load from type '{0}'".format(type(self._path_or_bytes))) self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta self._model_meta = self._sess.model_meta self._providers = self._sess.get_providers()