def __init__(self,
onnx_bytes,
sess_options=None,
log_severity_level=4,
device=None):
if InferenceSession is None:
raise ImportError( # pragma: no cover
"onnxruntime is not available.")
self.log_severity_level = log_severity_level
if device is None:
self.device = get_ort_device('cpu')
else:
self.device = get_ort_device(device)
self.providers = device_to_providers(self.device)
set_default_logger_severity(3)
if sess_options is None:
self.so = SessionOptions()
self.so.log_severity_level = log_severity_level
self.sess = OrtInferenceSession(onnx_bytes,
sess_options=self.so,
providers=self.providers)
else:
self.so = sess_options
self.sess = OrtInferenceSession(onnx_bytes,
sess_options=sess_options,
providers=self.providers)
self.ro = RunOptions()
self.ro.log_severity_level = log_severity_level
self.ro.log_verbosity_level = log_severity_level
self.output_names = [o.name for o in self.get_outputs()]
def run(self, inputs, **kwargs): # type: (Any, **Any) -> Tuple[Any, ...]
"""
Computes the prediction.
See :meth:`onnxruntime.InferenceSession.run`.
"""
options = RunOptions()
for k, v in kwargs.items():
if hasattr(options, k):
setattr(options, k, v)
if isinstance(inputs, list):
inps = {}
for i, inp in enumerate(self._session.get_inputs()):
inps[inp.name] = inputs[i]
outs = self._session.run(None, inps, options)
if isinstance(outs, list):
return outs
else:
output_names = [o.name for o in self._session.get_outputs()]
return [outs[name] for name in output_names]
else:
inp = self._session.get_inputs()
if len(inp) != 1:
raise RuntimeError("Model expect {0} inputs".format(len(inp)))
inps = {inp[0].name: inputs}
return self._session.run(None, inps, options)
def __setstate__(self, state):
"""
Overwrites getstate to get rid of InferenceSession.
"""
for k, v in state.items():
if k == 'ro_':
self.ro_ = RunOptions()
elif not k.endswith('_onnx_') and not k.endswith('_sess_'):
setattr(self, k, v)
so = SessionOptions()
so.log_severity_level = 4
for k, v in state.items():
if k.endswith('_onnx_'):
setattr(self, k, onnx.load(BytesIO(v)))
k2 = k.replace("onnx", "sess")
prov = state[k2]
setattr(
self, k2,
InferenceSession(getattr(self, k).SerializeToString(),
so,
providers=prov))
for k, v in state.items():
if k.endswith('_bind_'):
k2 = k[:-5]
setattr(self, k, getattr(self, k2).io_binding()._iobinding)
elif k.endswith('_binds_'):
k2 = k[:-6]
n = v
setattr(self, k, [
getattr(self, k2).io_binding()._iobinding for i in range(n)
])
self.cache_in_ = {}
self.cache_out_ = {}
return self
def __init__(self, onnx_data, runtime):
"""
@param onnx_data :epkg:`ONNX` model or data
@param runtime runtime to be used,
mostly :epkg:`onnxruntime`
"""
if runtime != 'onnxruntime1':
raise NotImplementedError(
"runtime '{}' is not implemented.".format(runtime))
if hasattr(onnx_data, 'SerializeToString'):
onnx_data = onnx_data.SerializeToString()
self.runtime = runtime
sess_options = SessionOptions()
self.run_options = RunOptions()
try:
sess_options.session_log_severity_level = 3
# sess_options.sessions_log_verbosity_level = 0
except AttributeError: # pragma: no cover
# onnxruntime not recent enough.
pass
try:
self.run_options.run_log_severity_level = 3
# self.run_options.run_log_verbosity_level = 0
except AttributeError: # pragma: no cover
# onnxruntime not recent enough.
pass
self.sess = InferenceSession(onnx_data, sess_options=sess_options)
def _init_next(self):
if self.enable_logging:
self._logger = logging.getLogger("onnxcustom")
else:
self._logger = None # pragma: no cover
if self.run_options is None:
self.run_options = RunOptions()
self.run_options.training_mode = True
if self.graph_builder_config is None:
initializer_names = [
i.name for i in self.onnx_model.graph.initializer
]
input_names = [i.name for i in self.onnx_model.graph.input]
config = OrtModuleGraphBuilderConfiguration()
config.initializer_names = [
init for init in initializer_names
if init in self.weights_to_train
]
config.initializer_names_to_train = self.weights_to_train
config.input_names_require_grad = input_names
config.build_gradient_graph = True
if (len(config.initializer_names) != # noqa
len(config.initializer_names_to_train)):
raise RuntimeError( # pragma: no cover
"Unable to automatically fill "
"OrtModuleGraphBuilderConfiguration, mismatch between "
"%r and %r (initializer_names=%r)." %
(config.initializer_names,
config.initializer_names_to_train, initializer_names))
p = TrainingGraphTransformerConfiguration()
config.graph_transformer_config = p
# config.enable_caching = True
# config.loglevel =
# config.use_memory_efficient_gradient = True
self.graph_builder_config = config
attributes = self._create_onnx_graphs()
attributes['__doc__'] = (
"Inherits from @see cl OrtGradientForwardBackwardFunction.")
attributes['__module__'] = (
OrtGradientForwardBackwardFunction.__module__)
self.cls_type_ = type(self.class_name,
(OrtGradientForwardBackwardFunction, ),
attributes)
def run_model(cls, model, inputs, device=None, **kwargs):
"""
Compute the prediction.
:param model: :class:`onnxruntime.InferenceSession` returned
by function *prepare*
:param inputs: inputs
:param device: requested device for the computation,
None means the default one which depends on
the compilation settings
:param kwargs: see :class:`onnxruntime.RunOptions`
:return: predictions
"""
rep = cls.prepare(model, device, **kwargs)
options = RunOptions()
for k, v in kwargs.items():
if hasattr(options, k):
setattr(options, k, v)
return rep.run(inputs, options)
def __init__(self, onnx_data, runtime, runtime_options=None):
"""
@param onnx_data :epkg:`ONNX` model or data
@param runtime runtime to be used,
mostly :epkg:`onnxruntime`
@param runtime_options runtime options
"""
if runtime != 'onnxruntime1':
raise NotImplementedError( # pragma: no cover
"runtime '{}' is not implemented.".format(runtime))
if hasattr(onnx_data, 'SerializeToString'):
onnx_data = onnx_data.SerializeToString()
self.runtime = runtime
sess_options = SessionOptions()
self.run_options = RunOptions()
try:
sess_options.sessions_log_verbosity_level = 0
except AttributeError: # pragma: no cover
# onnxruntime not recent enough.
pass
try:
self.run_options.run_log_verbosity_level = 0
except AttributeError: # pragma: no cover
# onnxruntime not recent enough.
pass
if (runtime_options is not None
and runtime_options.get('disable_optimisation', False)):
sess_options.graph_optimization_level = (
GraphOptimizationLevel.ORT_ENABLE_ALL)
try:
self.sess = InferenceSession(onnx_data, sess_options=sess_options)
except (OrtFail, OrtNotImplemented, OrtInvalidGraph,
OrtInvalidArgument, OrtRuntimeException, RuntimeError) as e:
raise RuntimeError(
"Unable to create InferenceSession due to '{}'\n{}.".format(
e, display_onnx(onnx.load(BytesIO(onnx_data))))) from e
def __init__(self):
BaseLearningOnnx.__init__(self)
self.ro_ = RunOptions()
def _init(self, variables=None):
"""
Initializes the node.
@param variables registered variables created by previous operators
The current implementation for operator *Scan*
only works for matrices.
"""
try:
self.alg_class = getattr(alg2, 'Onnx' + self.onnx_node.op_type)
except AttributeError:
self.alg_class = getattr(alg, 'Onnx' + self.onnx_node.op_type)
inputs = list(self.onnx_node.input)
self.mapping, self.inputs = self._name_mapping(inputs)
self.outputs = list(self.onnx_node.output)
options = self.options.copy()
target_opset = options.pop('target_opset', None)
domain = options.pop('domain', None)
disable_optimisation = options.pop('disable_optimisation', False)
ir_version = options.pop('ir_version', None)
if domain == '' and target_opset < 9:
# target_opset should be >= 9 not {} for main domain.
# We assume it was the case when the graph was created.
pass
if self.onnx_node.op_type == 'ConstantOfShape':
for k in options:
v = options[k]
if isinstance(v, numpy.ndarray):
options[k] = make_tensor(k,
self._guess_proto_type(v.dtype),
v.shape, v.tolist())
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
try:
self.onnx_ = self.inst_.to_onnx(inputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}".
format(self.onnx_))
except AttributeError as e: # pragma: no cover
# older version of skl2onnx
self.onnx_ = self.inst_.to_onnx(inputs)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError(
"Probable issue as one dimension is null.\n--\n{}".
format(self.onnx_)) from e
forced = False
elif self.onnx_node.op_type == 'Scan':
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
inputs = [(name, cl.__class__([None, None]))
for (name, cl) in inputs]
outputs = [(name, cl.__class__([None, None]))
for (name, cl) in outputs]
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}".format(
self.onnx_))
forced = True
else:
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
domain=domain,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
try:
self.onnx_ = self.inst_.to_onnx(inputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}\n---\n{}"
.format(self.onnx_, inputs))
forced = False
except (RuntimeError, ValueError):
# Let's try again by forcing output types.
forced = True
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}".
format(self.onnx_)) from e
if len(self.onnx_.graph.output) != len(self.outputs):
# Something is wrong, falls back to default plan.
forced = True
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError(
"Probable issue as one dimension is null.\n--\n{}".format(
self.onnx_))
else:
lo = list(self.onnx_.graph.output)
outputs = proto2vars(lo)
sess_options = SessionOptions()
self.run_options = RunOptions()
try:
sess_options.session_log_severity_level = 3
# sess_options.sessions_log_verbosity_level = 0
except AttributeError:
# onnxruntime not recent enough.
pass
try:
self.run_options.run_log_severity_level = 3
# self.run_options.run_log_verbosity_level = 0
except AttributeError:
# onnxruntime not recent enough.
pass
if ir_version is not None:
self.onnx_.ir_version = ir_version
if disable_optimisation:
sess_options.graph_optimization_level = (
GraphOptimizationLevel.ORT_DISABLE_ALL)
try:
self.sess_ = InferenceSession(self.onnx_.SerializeToString(),
sess_options=sess_options)
except (RuntimeError, OrtNotImplemented, OrtInvalidGraph,
OrtFail) as e:
raise RuntimeError(
"Unable to load node '{}' (output type was {})\n{}".format(
self.onnx_node.op_type,
"guessed" if forced else "inferred", self.onnx_)) from e
self.typed_outputs_ = outputs
def _init(self, variables=None):
"""
Initializes the node.
@param variables registered variables created by previous operators
The current implementation for operator *Scan*
only works for matrices.
"""
self.alg_class = getattr(alg, 'Onnx' + self.onnx_node.op_type)
inputs = list(self.onnx_node.input)
self.mapping, self.inputs = self._name_mapping(inputs)
self.outputs = list(self.onnx_node.output)
options = self.options.copy()
target_opset = options.pop('target_opset', None)
if self.onnx_node.op_type == 'ConstantOfShape':
for k in options:
v = options[k]
if isinstance(v, numpy.ndarray):
options[k] = make_tensor(k,
self._guess_proto_type(v.dtype),
v.shape, v.tolist())
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
self.onnx_ = self.inst_.to_onnx(inputs,
target_opset=target_opset,
dtype=self.dtype)
forced = False
elif self.onnx_node.op_type == 'Scan':
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
inputs = [(name, cl.__class__([None, None]))
for (name, cl) in inputs]
outputs = [(name, cl.__class__([None, None]))
for (name, cl) in outputs]
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
dtype=self.dtype)
forced = True
else:
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
try:
self.onnx_ = self.inst_.to_onnx(inputs,
target_opset=target_opset,
dtype=self.dtype)
forced = False
except (RuntimeError, ValueError):
# Let's try again by forcing output types.
forced = True
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
dtype=self.dtype)
if len(self.onnx_.graph.output) != self.outputs:
# Something is wrong, falls back to default plan.
forced = True
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
dtype=self.dtype)
sess_options = SessionOptions()
self.run_options = RunOptions()
try:
sess_options.session_log_severity_level = 3
# sess_options.sessions_log_verbosity_level = 0
except AttributeError:
# onnxruntime not recent enough.
pass
try:
self.run_options.run_log_severity_level = 3
# self.run_options.run_log_verbosity_level = 0
except AttributeError:
# onnxruntime not recent enough.
pass
try:
self.sess_ = InferenceSession(self.onnx_.SerializeToString(),
sess_options=sess_options)
except RuntimeError as e:
raise RuntimeError(
"Unable to load node '{}' (output type was {})\n{}".format(
self.onnx_node.op_type,
"guessed" if forced else "inferred", self.onnx_)) from e
self.typed_outputs_ = outputs