def infer():
# Step1：初始化ONNXRuntime库并配置相应参数, 并进行预测
# 加载ONNX模型
sess = InferenceSession(FLAGS.onnx_file)
# define transforms
input_shape = sess.get_inputs()[0].shape[2:]
eval_transforms = ClassificationPresetEval(
crop_size=input_shape, resize_size=FLAGS.crop_size)
# 准备输入
with open(FLAGS.img_path, 'rb') as f:
img = Image.open(f).convert('RGB')
img = eval_transforms(img)
img = np.expand_dims(img, axis=0)
# 模型预测
ort_outs = sess.run(output_names=None,
input_feed={sess.get_inputs()[0].name: img})
output = ort_outs[0]
class_id = output.argmax()
prob = output[0][class_id]
print("ONNXRuntime predict: ")
print(f"class_id: {class_id}, prob: {prob}")
def test_dict_vectorizer_rfr(self):
this = os.path.abspath(os.path.dirname(__file__))
data = os.path.join(this, "data", "pipeline_vectorize.onnx")
sess = InferenceSession(data)
input_name = sess.get_inputs()[0].name
self.assertEqual(input_name, "float_input")
input_type = str(sess.get_inputs()[0].type)
self.assertEqual(input_type, "map(int64,tensor(float))")
input_shape = sess.get_inputs()[0].shape
self.assertEqual(input_shape, [])
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "variable1")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, "tensor(float)")
output_shape = sess.get_outputs()[0].shape
self.assertEqual(output_shape, [1, 1])
x = {0: 25.0, 1: 5.13, 2: 0.0, 3: 0.453, 4: 5.966}
res = sess.run([output_name], {input_name: x})
model_onnx = onnx.load(data)
oinf = OnnxInference(model_onnx, runtime='onnxruntime1')
res2 = oinf.run({input_name: x})
x = {k: numpy.float32(v) for k, v in x.items()}
oinf = OnnxInference(model_onnx, runtime='python')
res3 = oinf.run({input_name: [x]}) # , verbose=10, fLOG=print)
self.assertEqualFloat(res[0][0, 0], res2["variable1"][0, 0])
self.assertEqualFloat(res[0][0, 0], res3["variable1"][0])
def format_results(sess: rt.InferenceSession, data: list) -> dict:
input_name = sess.get_inputs()[0].name
input_shape = sess.get_inputs()[0].shape
label_name = sess.get_outputs()[0].name
return {
'input_name': input_name,
'input_shape': input_shape[1:],
'output_name': label_name,
'prediction': data,
'createdAt': datetime.now(timezone.utc).astimezone().isoformat()
}
class OnnxModelLoader:
def __init__(self, onnx_path: str):
"""
Class for loading ONNX models to inference on CPU.
:param onnx_path: path to ONNX model file (*.onnx file).
"""
self.onnx_path = onnx_path
self.sess = InferenceSession(self.onnx_path,
providers=['CPUExecutionProvider'])
# In current case model always has exactly one input and one output.
self.input_name = [x.name for x in self.sess.get_inputs()][0]
self.output_name = [x.name for x in self.sess.get_outputs()][0]
def inference(self, inputs: np.ndarray) -> np.ndarray:
"""
Run inference.
:param inputs: input numpy array.
:return: output numpy array.
"""
outputs = self.sess.run(
[self.output_name],
input_feed={self.input_name: np.float32(inputs)})
return outputs[0]
def run_inference(sess: rt.InferenceSession,
TRUNCATION: float = 0.7,
seed=None) -> np.array:
# return raw array from Generator
# really badly set numpy seed from string if provided
# note that `hash()` would not work here, as it is itself initialized from a random state
# it would be possible to force hash() to be set from a static type, but would require an additional python file
# to set the environment seed... So equally awful
random.seed(seed) # this may be a string
np.random.seed(
random.randint(0, 2**32 - 1)
) # use python random state to generate numpy random state repeatably, lol
# randomized generator inputs
latents = np.random.randn(1, 512).astype(np.float32)
truncation = np.array([TRUNCATION]).astype(np.float32)
input_name_a = sess.get_inputs()[0].name
input_name_b = sess.get_inputs()[1].name
label_name = sess.get_outputs()[0].name
pred = sess.run([label_name], {
input_name_a: latents,
input_name_b: truncation
})[0]
return pred
def test_xgboost_classifier_i5450(self):
iris = load_iris()
X, y = iris.data, iris.target
X_train, X_test, y_train, y_test = train_test_split(X,
y,
random_state=10)
clr = XGBClassifier(objective="multi:softmax",
max_depth=1,
n_estimators=2)
clr.fit(X_train,
y_train,
eval_set=[(X_test, y_test)],
early_stopping_rounds=40)
initial_type = [('float_input', FloatTensorType([None, 4]))]
onx = convert_xgboost(clr, initial_types=initial_type)
sess = InferenceSession(onx.SerializeToString())
input_name = sess.get_inputs()[0].name
label_name = sess.get_outputs()[1].name
predict_list = [1., 20., 466., 0.]
predict_array = np.array(predict_list).reshape(
(1, -1)).astype(np.float32)
pred_onx = sess.run([label_name], {input_name: predict_array})[0]
pred_xgboost = sessresults = clr.predict_proba(predict_array)
bst = clr.get_booster()
bst.dump_model('dump.raw.txt')
dump_data_and_model(
X_test.astype(np.float32) + 1e-5,
clr,
onx,
allow_failure=
"StrictVersion(onnx.__version__) < StrictVersion('1.3.0')",
basename="XGBClassifierIris")
def _predict_with_onnx(model, X):
session = InferenceSession(model.SerializeToString())
output_names = [s_output.name for s_output in session.get_outputs()]
input_names = [s_input.name for s_input in session.get_inputs()]
if len(input_names) > 1:
raise RuntimeError(
"Test expects one input. Found multiple inputs: %r."
"" % input_names)
input_name = input_names[0]
return session.run(output_names, {input_name: X})[0][:, 0]
def get_io_numpy_type_map(
ort_session: InferenceSession) -> Dict[str, numpy.dtype]:
"""Create a mapping from input/output name to numpy data type"""
name_to_numpy_type = {}
for input in ort_session.get_inputs():
name_to_numpy_type[input.name] = TypeHelper.ort_type_to_numpy_type(
input.type)
for output in ort_session.get_outputs():
name_to_numpy_type[
output.name] = TypeHelper.ort_type_to_numpy_type(output.type)
return name_to_numpy_type
def onnx_predict(sess: InferenceSession, x: np.ndarray):
"""
use ONNX runtime session to predict result
:param sess: ONNX runtime session
:param x: input ndarray
:return: predicted result
"""
x = x.reshape((-1, 1, 28, 28))
input_name = sess.get_inputs()[0].name
label_name = sess.get_outputs()[0].name
pred = sess.run([label_name], {input_name: x.astype("float32")})[0]
return np.argmax(pred, axis=1)[0]
def predict(ort_session: ort.InferenceSession, im_array: np.ndarray) -> Image:
sample = preprocess(im_array)
inputs_test = np.expand_dims(sample["image"], 0).astype(np.float32)
ort_inputs = {ort_session.get_inputs()[0].name: inputs_test}
ort_outs = ort_session.run(None, ort_inputs)
d1 = ort_outs[0]
pred = d1[:, 0, :, :]
predict = np.squeeze(norm_pred(pred))
img = Image.fromarray(predict * 255).convert("RGB")
return img
def test_variable_names(self):
pipeline = Pipeline([("passthrough", Passthrough())])
initial_types = [("input", FloatTensorType([None, 2]))]
model_onnx = convert_sklearn(pipeline,
initial_types=initial_types,
target_opset=TARGET_OPSET,
verbose=0)
self.assertIn('Identity', str(model_onnx))
x = np.array([0, 1, 1, 0], dtype=np.float32).reshape((-1, 2))
sess = InferenceSession(model_onnx.SerializeToString())
name = sess.get_inputs()[0].name
got = sess.run(None, {name: x})
assert_almost_equal(x, got[0])
class SimplifiedOnnxInference:
"Simple wrapper around InferenceSession which imitates OnnxInference."
def __init__(self, ort):
from onnxruntime import InferenceSession
self.sess = InferenceSession(ort)
@property
def input_names(self):
"Returns InferenceSession input names."
return [_.name for _ in self.sess.get_inputs()]
def run(self, input):
"Calls InferenceSession.run."
return self.sess.run(None, input)
def test_pipeline_make_column_selector(self):
X = pandas.DataFrame({
'city': ['London', 'London', 'Paris', 'Sallisaw'],
'rating': [5, 3, 4, 5]})
X['rating'] = X['rating'].astype(numpy.float32)
ct = make_column_transformer(
(StandardScaler(), make_column_selector(
dtype_include=numpy.number)),
(OneHotEncoder(), make_column_selector(
dtype_include=object)))
expected = ct.fit_transform(X)
onx = to_onnx(ct, X, target_opset=TARGET_OPSET)
sess = InferenceSession(onx.SerializeToString())
names = [i.name for i in sess.get_inputs()]
got = sess.run(None, {names[0]: X[names[0]].values.reshape((-1, 1)),
names[1]: X[names[1]].values.reshape((-1, 1))})
assert_almost_equal(expected, got[0])
def verify_onnx(model: T5EncoderDecoderInit, ort_session: InferenceSession, device: torch.device, max_cases=4):
""" Compare the result from PyTorch and OnnxRuntime to verify the ONNX model is good.
"""
ort_inputs = ort_session.get_inputs()
use_decoder_input_ids = len(ort_inputs) == 3
test_cases = [(4, 11), (1, 2), (3, 1), (8, 5)]
test_cases_max_diff = []
for (batch_size, encode_sequence_length) in test_cases[:max_cases]:
inputs = T5EncoderDecoderInitInputs.create_dummy(model.config,
batch_size,
encode_sequence_length,
use_decoder_input_ids=use_decoder_input_ids,
device=device)
ort_outputs = T5EncoderDecoderInitHelper.onnxruntime_inference(ort_session, inputs)
# Run inference of PyTorch model
input_list = inputs.to_list()
torch_outputs = model(*input_list)
assert (torch_outputs[0].cpu().numpy().shape == ort_outputs[0].shape)
max_diff = numpy.amax(numpy.abs(torch_outputs[0].cpu().numpy() - ort_outputs[0]))
logger.debug(f"logits max_diff={max_diff}")
max_diff_all = max_diff
assert (torch_outputs[1].cpu().numpy().shape == ort_outputs[1].shape)
max_diff = numpy.amax(numpy.abs(torch_outputs[1].cpu().numpy() - ort_outputs[1]))
logger.debug(f"encoder_hidden_states max_diff={max_diff}")
max_diff_all = max(max_diff_all, max_diff)
for i in range(2 * model.config.num_layers):
max_diff = numpy.amax(numpy.abs(torch_outputs[2][i].cpu().numpy() - ort_outputs[2 + i]))
logger.debug(f"self attention past state {i} max_diff={max_diff}")
for i in range(2 * model.config.num_layers):
max_diff = numpy.amax(
numpy.abs(torch_outputs[3][i].cpu().numpy() - ort_outputs[2 + 2 * model.config.num_layers + i]))
logger.debug(f"cross attention past state {i} max_diff={max_diff}")
max_diff_all = max(max_diff_all, max_diff)
test_cases_max_diff.append(max_diff_all)
logger.info(
f"batch_size={batch_size} encode_sequence_length={encode_sequence_length}, max_diff={max_diff_all}")
return max(test_cases_max_diff)
def _test_lgbm(self, X, model, extra_config={}):
# Create ONNX-ML model
onnx_ml_model = convert_model(
model, 'lgbm-onnxml',
[("input", FloatTensorType([X.shape[0], X.shape[1]]))])[0]
# Create ONNX model
onnx_model = convert_model(
model,
'lgbm-onnx',
[("input", FloatTensorType([X.shape[0], X.shape[1]]))],
without_onnx_ml=True)[0]
try:
from onnxruntime import InferenceSession
except ImportError:
# onnxruntime not installed (python 2.7)
return
# Get the predictions for the ONNX-ML model
session = InferenceSession(onnx_ml_model.SerializeToString())
output_names = [
session.get_outputs()[i].name
for i in range(len(session.get_outputs()))
]
onnx_ml_pred = [[] for i in range(len(output_names))]
inputs = {session.get_inputs()[0].name: X}
pred = session.run(output_names, inputs)
for i in range(len(output_names)):
if output_names[i] == "label":
onnx_ml_pred[1] = pred[i]
else:
onnx_ml_pred[0] = pred[i]
# Get the predictions for the ONNX model
session = InferenceSession(onnx_model.SerializeToString())
onnx_pred = [[] for i in range(len(output_names))]
pred = session.run(output_names, inputs)
for i in range(len(output_names)):
if output_names[i] == "label":
onnx_pred[1] = pred[i]
else:
onnx_pred[0] = pred[i]
return onnx_ml_pred, onnx_pred, output_names
class OnnxModelLoader:
def __init__(self, onnx_path: str) -> None:
"""
Class for loading ONNX models to inference on CPU. CPU inference is very effective using onnxruntime.
:param onnx_path: path to ONNX model file (*.onnx file).
"""
self.sess = InferenceSession(onnx_path,
providers=['CPUExecutionProvider'])
self.input_name = [x.name for x in self.sess.get_inputs()][0]
self.output_names = [x.name for x in self.sess.get_outputs()]
def inference(self, inputs: np.ndarray) -> List[np.ndarray]:
"""
Run inference.
:param inputs: list of arguments, order must match names in input_names.
:return: list of outputs.
"""
return self.sess.run(self.output_names,
input_feed={self.input_name: inputs})
def benchmark(name,
onx,
fct_numpy,
*args,
dims=(1, 10, 100, 200, 500, 1000, 2000, 10000)):
sess = InferenceSession(onx.SerializeToString())
device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0)
names = [i.name for i in sess.get_inputs()]
out_names = [o.name for o in sess.get_outputs()]
if len(names) != len(args):
raise RuntimeError(f"Size mismatch {len(names)} != {len(args)}.")
rows = []
for dim in tqdm(dims):
new_args = [reshape(a, dim) for a in args]
ortvalues = [
C_OrtValue.ortvalue_from_numpy(a, device) for a in new_args
]
ms = measure_time(lambda: fct_numpy(*new_args), repeat=50, number=100)
ms.update(dict(name=name, impl='numpy', dim=dim))
rows.append(ms)
inps = {n: a for n, a in zip(names, new_args)}
ms = measure_time(lambda: sess.run(None, inps))
ms.update(dict(name=name, impl='sess', dim=dim))
rows.append(ms)
bind = SessionIOBinding(sess._sess)
ms = measure_time(lambda: bind_and_run(sess._sess, bind, names,
ortvalues, out_names, device))
ms.update(dict(name=name, impl='bind_run', dim=dim))
rows.append(ms)
ms = measure_time(lambda: nobind_just_run(sess._sess, bind))
ms.update(dict(name=name, impl='run', dim=dim))
rows.append(ms)
return rows
def predict_with_onnxruntime(onx, X):
sess = InferenceSession(onx.SerializeToString())
input_name = sess.get_inputs()[0].name
res = sess.run(None, {input_name: X.astype(np.float32)})
return res[0]
def forward_no_training(self, exc=None, verbose=False):
if exc is None:
exc = __name__ != '__main__'
from onnxruntime.capi._pybind_state import (OrtValue as C_OrtValue,
OrtDevice as C_OrtDevice,
OrtMemType)
from onnxruntime.capi._pybind_state import (OrtValueVector)
from onnxcustom.training.ortgradient import OrtGradientForwardBackward
X, y = make_regression( # pylint: disable=W0632
100, n_features=10, bias=2)
X = X.astype(numpy.float32)
y = y.astype(numpy.float32)
X_train, X_test, y_train, _ = train_test_split(X, y)
reg = LinearRegression()
reg.fit(X_train, y_train)
reg.coef_ = reg.coef_.reshape((1, -1))
onx = to_onnx(reg,
X_train,
target_opset=opset,
black_op={'LinearRegressor'})
# starts testing
if verbose:
print("[forward_no_training] start testing")
if exc:
if verbose:
print("[forward_no_training] check exception")
self.assertRaise(
lambda: OrtGradientForwardBackward(
onx, debug=True, enable_logging=True, providers=['NONE']),
ValueError)
if verbose:
print("[forward_no_training] instantiate")
forback = OrtGradientForwardBackward(onx,
debug=True,
enable_logging=True)
self.assertEqual(repr(forback), "OrtGradientForwardBackward(...)")
self.assertTrue(hasattr(forback, 'cls_type_'))
self.assertEqual(forback.cls_type_._onx_inp,
['X', 'coef', 'intercept'])
self.assertEqual(forback.cls_type_._onx_out,
['X_grad', 'coef_grad', 'intercept_grad'])
self.assertEqual(forback.cls_type_._weights_to_train,
['coef', 'intercept'])
self.assertEqual(forback.cls_type_._grad_input_names,
['X', 'coef', 'intercept'])
self.assertEqual(forback.cls_type_._input_names, ['X'])
self.assertEqual(forback.cls_type_._bw_fetches_names,
['X_grad', 'coef_grad', 'intercept_grad'])
self.assertEqual(forback.cls_type_._output_names, ['variable'])
if verbose:
print("[forward_no_training] expected prediction")
expected = reg.predict(X_test)
coef = reg.coef_.astype(numpy.float32).reshape((-1, 1))
intercept = numpy.array([reg.intercept_], dtype=numpy.float32)
if verbose:
print("[forward_no_training] InferenceSession")
providers = device_to_providers('cpu')
sess0 = InferenceSession(onx.SerializeToString(), providers=providers)
inames = [i.name for i in sess0.get_inputs()] # pylint: disable=E1101
self.assertEqual(inames, ['X'])
got = sess0.run(None, {'X': X_test})
self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4)
if verbose:
print("[forward_no_training] evaluation")
sess_eval = forback.cls_type_._sess_eval # pylint: disable=E1101
inames = [i.name for i in sess_eval.get_inputs()]
self.assertEqual(inames, ['X', 'coef', 'intercept'])
got = sess_eval.run(None, {
'X': X_test,
'coef': coef,
'intercept': intercept
})
self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4)
# OrtValue
if verbose:
print("[forward_no_training] OrtValue")
inst = forback.new_instance()
device = C_OrtDevice(C_OrtDevice.cpu(), OrtMemType.DEFAULT, 0)
# list of OrtValues
inputs = []
for a in [X_test, coef, intercept]:
inputs.append(C_OrtValue.ortvalue_from_numpy(a, device))
got_ort = inst.forward(inputs)
got = [v.numpy() for v in got_ort]
self.assertEqual(len(got), 1)
self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4)
# OrtValueVector
if verbose:
print("[forward_no_training] OrtValueVector")
inputs = OrtValueVector()
for a in [X_test, coef, intercept]:
inputs.push_back(C_OrtValue.ortvalue_from_numpy(a, device))
got = inst.forward(inputs)
self.assertEqual(len(got), 1)
self.assertEqualArray(expected.ravel(),
got[0].numpy().ravel(),
decimal=4)
# numpy
if verbose:
print("[forward_no_training] numpy")
inputs = [X_test, coef, intercept]
got = inst.forward(inputs)
self.assertEqual(len(got), 1)
self.assertEqualArray(expected.ravel(),
got[0].numpy().ravel(),
decimal=4)
if verbose:
print("[forward_no_training] end")
def test_forward_no_training_pickle(self):
from onnxruntime.capi._pybind_state import (OrtValue as C_OrtValue,
OrtMemType, OrtDevice as
C_OrtDevice)
from onnxruntime.capi._pybind_state import (OrtValueVector)
from onnxcustom.training.ortgradient import OrtGradientForwardBackward
X, y = make_regression( # pylint: disable=W0632
100, n_features=10, bias=2)
X = X.astype(numpy.float32)
y = y.astype(numpy.float32)
X_train, X_test, y_train, _ = train_test_split(X, y)
reg = LinearRegression()
reg.fit(X_train, y_train)
reg.coef_ = reg.coef_.reshape((1, -1))
onx = to_onnx(reg,
X_train,
target_opset=opset,
black_op={'LinearRegressor'})
forback0 = OrtGradientForwardBackward(onx, debug=True)
st = io.BytesIO()
pickle.dump(forback0, st)
st2 = io.BytesIO(st.getvalue())
forback = pickle.load(st2)
self.assertTrue(hasattr(forback, 'cls_type_'))
self.assertEqual(forback.cls_type_._onx_inp,
['X', 'coef', 'intercept'])
self.assertEqual(forback.cls_type_._onx_out,
['X_grad', 'coef_grad', 'intercept_grad'])
self.assertEqual(forback.cls_type_._weights_to_train,
['coef', 'intercept'])
self.assertEqual(forback.cls_type_._grad_input_names,
['X', 'coef', 'intercept'])
self.assertEqual(forback.cls_type_._input_names, ['X'])
self.assertEqual(forback.cls_type_._bw_fetches_names,
['X_grad', 'coef_grad', 'intercept_grad'])
self.assertEqual(forback.cls_type_._output_names, ['variable'])
expected = reg.predict(X_test)
coef = reg.coef_.astype(numpy.float32).reshape((-1, 1))
intercept = numpy.array([reg.intercept_], dtype=numpy.float32)
providers = device_to_providers('cpu')
sess0 = InferenceSession(onx.SerializeToString(), providers=providers)
inames = [i.name for i in sess0.get_inputs()]
self.assertEqual(inames, ['X'])
got = sess0.run(None, {'X': X_test})
self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4)
sess_eval = forback.cls_type_._sess_eval # pylint: disable=W0212
inames = [i.name for i in sess_eval.get_inputs()]
self.assertEqual(inames, ['X', 'coef', 'intercept'])
got = sess_eval.run(None, {
'X': X_test,
'coef': coef,
'intercept': intercept
})
self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4)
# OrtValue
inst = forback.new_instance()
inputs = []
device = C_OrtDevice(C_OrtDevice.cpu(), OrtMemType.DEFAULT, 0)
for a in [X_test, coef, intercept]:
inputs.append(C_OrtValue.ortvalue_from_numpy(a, device))
got_ort = inst.forward(inputs)
got = [v.numpy() for v in got_ort]
self.assertEqual(len(got), 1)
self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4)
# OrtValueVector
inputs = OrtValueVector()
for a in [X_test, coef, intercept]:
inputs.push_back(C_OrtValue.ortvalue_from_numpy(a, device))
got = inst.forward(inputs)
self.assertEqual(len(got), 1)
self.assertEqualArray(expected.ravel(),
got[0].numpy().ravel(),
decimal=4)
# numpy
inputs = [X_test, coef, intercept]
got = inst.forward(inputs)
self.assertEqual(len(got), 1)
self.assertEqualArray(expected.ravel(),
got[0].numpy().ravel(),
decimal=4)
def get_input_type(ort_session: InferenceSession, name: str) -> str:
for i, input in enumerate(ort_session.get_inputs()):
if input.name == name:
return input.type
raise ValueError(f"input name {name} not found")
def _create_onnx_graphs(self):
"""
Creates forward and backward ONNX graph.
The new class has the following attributes:
* `__doc__`: doc string
* `__module__`: module name (this file)
* `_run_options`: see :epkg:`RunOptions`
* `_sess`: :epkg:`InferenceSession` with the original graph
* `_sess_eval`: :epkg:`InferenceSession` on the graph
with weights as inputs
* `_training_agent`: :epkg:`TrainingAgent`
* `_cache`: :epkg:`OrtValueCache`
* `_logger`: logger
* `_input_names`: input names
* `_debug`: use debug mode
* `_grad_input_names`: gradient input names
* `_output_names`: output names
* `_weights_to_train`: names of the weights to train
Training attributes
* `_bw_fetches_names`: bw_fetches_names,
* `_fw_outputs_device_info`: fw_outputs_device_info,
* `_bw_outputs_device_info`: bw_outputs_device_info,
* `_fw_no_grad_output_device_info`: fw_no_grad_output_device_info,
* `_graph_info`: graph_info}
Additional attributes added if *keep_model* is True:
* `_trained_onnx`: ONNX graph for the gradient
* `_optimized_pre_grad_model`: evaluation ONNX graph taking
weights as inputs
* `_graph_builder`: :epkg:`OrtModuleGraphBuilder`
"""
logger = self._logger
if logger is not None:
logger.info("[OrtGradientForwardBackward] create training onnx")
logger.info("[OrtGradientForwardBackward] input_names=%r",
self.input_names)
logger.info("[OrtGradientForwardBackward] output_names=%r",
self.output_names)
logger.info("[OrtGradientForwardBackward] weights_to_train=%r",
self.weights_to_train)
builder = OrtModuleGraphBuilder()
if logger is not None:
cf = self.graph_builder_config.graph_transformer_config
cfp = cf.propagate_cast_ops_config
logger.info("[OrtGradientForwardBackward] "
"OrtModuleGraphBuilder.initialize")
logger.info(
"[OrtGradientForwardBackward] graph_builder_config=%s",
OrtGradientForwardBackward._repr_helper_(
self.graph_builder_config, indent=4))
logger.info(
"[OrtGradientForwardBackward] graph_builder_config."
"graph_transformer_config=%s",
OrtGradientForwardBackward._repr_helper_(cf, indent=4))
logger.info(
"[OrtGradientForwardBackward] graph_builder_config."
"graph_transformer_config.propagate_cast_ops_config=%s",
OrtGradientForwardBackward._repr_helper_(cfp, indent=4))
builder.initialize(self.onnx_model.SerializeToString(),
self.graph_builder_config)
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] OrtModuleGraphBuilder.build")
builder.build()
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] OrtModuleGraphBuilder.get_model")
train_onnx_model_serialized = builder.get_model()
optimized_pre_grad_model = builder.get_inference_optimized_model()
graph_info = builder.get_graph_info()
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] graph_info=%s",
OrtGradientForwardBackward._repr_helper_(graph_info, indent=4))
logger.info("[OrtGradientForwardBackward] create TrainSession")
logger.info(
"[OrtGradientForwardBackward] sess_options=%s",
OrtGradientForwardBackward._repr_helper_(self.sess_options,
indent=4))
logger.info("[OrtGradientForwardBackward] providers=%r",
self.providers)
sess = InferenceSession(train_onnx_model_serialized,
sess_options=self.sess_options,
provider_options=self.provider_options,
providers=self.providers)
if logger is not None:
logger.info("[OrtGradientForwardBackward] create InferenceSession")
sess_eval = InferenceSession(optimized_pre_grad_model,
sess_options=self.sess_options,
provider_options=self.provider_options,
providers=self.providers)
if logger is not None:
logger.info("[OrtGradientForwardBackward] create training agent")
grad_input_names = [obj.name for obj in sess.get_inputs()]
bw_fetches_names = [obj.name for obj in sess.get_outputs()]
fw_outputs_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(i),
OrtDevice.default_memory(), self.device_index)
for i in self.providers
]
bw_outputs_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(
self.providers[0]), OrtDevice.default_memory(),
self.device_index) for i in bw_fetches_names
]
fw_no_grad_output_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(
self.providers[0]), OrtDevice.default_memory(),
self.device_index) for i in self.output_names
]
try:
# onnxruntime>=1.12
training_agent = TrainingAgent(sess._sess, grad_input_names,
fw_outputs_device_info,
bw_fetches_names,
bw_outputs_device_info, 0)
except TypeError:
# onnxruntime<=1.11
training_agent = TrainingAgent(sess._sess, grad_input_names,
fw_outputs_device_info,
bw_fetches_names,
bw_outputs_device_info)
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] instantiate dynamic class %r",
self.class_name)
logger.info("[OrtGradientForwardBackward] weights_to_train=%r",
self.weights_to_train)
logger.info("[OrtGradientForwardBackward] grad_input_names=%r",
grad_input_names)
logger.info("[OrtGradientForwardBackward] bw_fetches_names=%r",
bw_fetches_names)
logger.info("[OrtGradientForwardBackward] device_index=%r",
self.device_index)
devices = list(fw_outputs_device_info)
while len(devices) < len(grad_input_names):
devices.append(devices[-1])
trained_onnx = onnx.load(BytesIO(train_onnx_model_serialized))
onnx_loss = onnx.load(BytesIO(optimized_pre_grad_model))
for i, node in enumerate(trained_onnx.graph.node):
if node.name == '':
node.name = "N%d" % i
for i, node in enumerate(onnx_loss.graph.node):
if node.name == '':
node.name = "N%d" % i
kwargs = {
'_run_options': self.run_options,
'_sess': sess,
'_sess_eval': sess_eval,
'_training_agent': training_agent,
'_cache': OrtValueCache(),
'_logger': logger,
'_input_names': self.input_names,
'_grad_input_names': grad_input_names,
'_output_names': self.output_names,
'_bw_fetches_names': bw_fetches_names,
'_fw_outputs_device_info': fw_outputs_device_info,
'_bw_outputs_device_info': bw_outputs_device_info,
'_fw_no_grad_output_device_info': fw_no_grad_output_device_info,
'_weights_to_train': list(sorted(self.weights_to_train)),
'_graph_info': graph_info,
#
'_trained_onnx': trained_onnx,
'_optimized_pre_grad_model': onnx_loss,
'_graph_builder': builder,
'_devices': devices,
'_debug': self.debug
}
graph = kwargs['_trained_onnx'].graph
kwargs.update({
'_onx_inp': [o.name for o in graph.input],
'_onx_out': [o.name for o in graph.output]
})
if len(kwargs['_onx_inp']) != len(kwargs['_onx_out']):
raise RuntimeError( # pragma: no cover
"Gradient input and output are inconsistant: "
"%r != %r" % (kwargs['_onx_inp'], kwargs['_onx_out']))
return kwargs
class OpRunOnnxRuntime:
"""
Unique operator which calls :epkg:`onnxruntime`
to compute predictions for one operator.
"""
def __init__(self,
onnx_node,
desc=None,
variables=None,
dtype=None,
**options):
"""
@param onnx_node :epkg:`onnx` node
@param desc internal representation
@param variables registered variables created by previous operators
@param dtype float computation type
@param options runtime options
"""
self._provider = 'onnxruntime'
self.onnx_node = onnx_node
self.desc = desc
self._schema = _schemas.get(onnx_node.op_type, None)
if desc is not None:
if 'atts' in desc:
for a, b in desc['atts'].items():
if not isinstance(b, dict) or 'value' not in b:
raise ValueError( # pragma: no cover
"Unexpected value {}.".format(b))
options[a] = b['value']
self.options = options
self.dtype = dtype
self._init(variables)
def _name_mapping(self, inputs):
mapping = {}
new_inputs = []
for name in inputs:
if name in mapping:
i = 0
new_name = "{}_{}".format(name, i)
while new_name in mapping:
i += 1
new_name = "{}_{}".format(name, i)
mapping[new_name] = name
new_inputs.append(new_name)
else:
new_inputs.append(name)
mapping[name] = name
return mapping, new_inputs
def _guess_proto_type(self, dtype):
if dtype == numpy.float32:
return TensorProto.FLOAT # pylint: disable=E1101
if dtype == numpy.float64:
return TensorProto.DOUBLE # pylint: disable=E1101
if dtype == numpy.int64:
return TensorProto.INT64 # pylint: disable=E1101
raise RuntimeError("Unable to guess type for dtype={}.".format(
dtype)) # pragma: no cover
def _init(self, variables=None):
"""
Initializes the node.
@param variables registered variables created by previous operators
The current implementation for operator *Scan*
only works for matrices.
"""
try:
self.alg_class = getattr(alg2, 'Onnx' + self.onnx_node.op_type)
except AttributeError:
self.alg_class = getattr(alg, 'Onnx' + self.onnx_node.op_type)
inputs = list(self.onnx_node.input)
self.mapping, self.inputs = self._name_mapping(inputs)
self.outputs = list(self.onnx_node.output)
options = self.options.copy()
target_opset = options.pop('target_opset', None)
domain = options.pop('domain', None)
disable_optimisation = options.pop('disable_optimisation', False)
ir_version = options.pop('ir_version', None)
if domain == '' and target_opset < 9:
# target_opset should be >= 9 not {} for main domain.
# We assume it was the case when the graph was created.
pass
if self.onnx_node.op_type == 'ConstantOfShape':
for k in options:
v = options[k]
if isinstance(v, numpy.ndarray):
options[k] = make_tensor(k,
self._guess_proto_type(v.dtype),
v.shape, v.tolist())
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
try:
self.onnx_ = self.inst_.to_onnx(inputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}".
format(self.onnx_))
except AttributeError as e: # pragma: no cover
# older version of skl2onnx
self.onnx_ = self.inst_.to_onnx(inputs)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError(
"Probable issue as one dimension is null.\n--\n{}".
format(self.onnx_)) from e
forced = False
elif self.onnx_node.op_type == 'Scan':
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
inputs = [(name, cl.__class__([None, None]))
for (name, cl) in inputs]
outputs = [(name, cl.__class__([None, None]))
for (name, cl) in outputs]
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}".format(
self.onnx_))
forced = True
else:
self.inst_ = self.alg_class(*self.inputs,
output_names=self.outputs,
op_version=target_opset,
domain=domain,
**options)
inputs = get_defined_inputs(self.inputs,
variables,
dtype=self.dtype)
try:
self.onnx_ = self.inst_.to_onnx(inputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}\n---\n{}"
.format(self.onnx_, inputs))
forced = False
except (RuntimeError, ValueError):
# Let's try again by forcing output types.
forced = True
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError( # pragma: no cover
"Probable issue as one dimension is null.\n--\n{}".
format(self.onnx_)) from e
if len(self.onnx_.graph.output) != len(self.outputs):
# Something is wrong, falls back to default plan.
forced = True
outputs = get_defined_outputs(self.outputs,
self.onnx_node,
inputs,
variables,
dtype=self.dtype)
self.onnx_ = self.inst_.to_onnx(inputs,
outputs=outputs,
target_opset=target_opset,
domain=domain)
if "dim_value: 0" in str(self.onnx_):
raise RuntimeError(
"Probable issue as one dimension is null.\n--\n{}".format(
self.onnx_))
else:
lo = list(self.onnx_.graph.output)
outputs = proto2vars(lo)
sess_options = SessionOptions()
self.run_options = RunOptions()
try:
sess_options.session_log_severity_level = 3
# sess_options.sessions_log_verbosity_level = 0
except AttributeError:
# onnxruntime not recent enough.
pass
try:
self.run_options.run_log_severity_level = 3
# self.run_options.run_log_verbosity_level = 0
except AttributeError:
# onnxruntime not recent enough.
pass
if ir_version is not None:
self.onnx_.ir_version = ir_version
if disable_optimisation:
sess_options.graph_optimization_level = (
GraphOptimizationLevel.ORT_DISABLE_ALL)
try:
self.sess_ = InferenceSession(self.onnx_.SerializeToString(),
sess_options=sess_options)
except (RuntimeError, OrtNotImplemented, OrtInvalidGraph,
OrtFail) as e:
raise RuntimeError(
"Unable to load node '{}' (output type was {})\n{}".format(
self.onnx_node.op_type,
"guessed" if forced else "inferred", self.onnx_)) from e
self.typed_outputs_ = outputs
def run(self, *args, **kwargs):
"""
Should be overwritten.
"""
inputs = {name: val for name, val in zip(self.inputs, args)}
try:
res = self.sess_.run(None, inputs, self.run_options)
except (RuntimeError, OrtInvalidArgument) as e: # pragma: no cover
dtypes = {k: v.dtype for k, v in inputs.items()}
shapes = {k: v.shape for k, v in inputs.items()}
exp = [_.name for _ in self.sess_.get_inputs()]
exp_types = [_.type for _ in self.sess_.get_inputs()]
raise RuntimeError(
"Predictions failed. List of inputs: {}, class={}"
"\ndtypes={}\nshapes={}\nexpected={}\nexpected={}\n"
"exception={}\n--ONNX--\n{}".format(list(sorted(inputs)),
self.alg_class, dtypes,
shapes, exp, exp_types, e,
self.onnx_)) from e
return tuple(res)
def infer_from_onnx(model_onnx, input_list):
sess = InferenceSession(model_onnx.SerializeToString())
input_name = sess.get_inputs()[0].name
pred_onx = sess.run(None, {input_name: numpy.array(input_list, numpy.float32)})
return pred_onx
from sklearn.linear_model import LogisticRegression
from skl2onnx.common.data_types import FloatTensorType, Int64TensorType
from skl2onnx import to_onnx
iris = load_iris()
X, y = iris.data, iris.target
X = X.astype(numpy.float32)
X_train, X_test, y_train, y_test = train_test_split(X, y)
clr = LogisticRegression(solver="liblinear")
clr.fit(X_train, y_train)
onx = to_onnx(clr, X, options={'zipmap': False})
sess = InferenceSession(onx.SerializeToString())
input_names = [i.name for i in sess.get_inputs()]
output_names = [o.name for o in sess.get_outputs()]
print("inputs=%r, outputs=%r" % (input_names, output_names))
print(sess.run(None, {input_names[0]: X_test[:2]}))
####################################
# Changes the input names
# +++++++++++++++++++++++
#
# It is possible to change the input name by using the
# parameter *initial_types*. However, the user must specify the input
# types as well.
onx = to_onnx(clr,
X,
options={'zipmap': False},
class OnnxTransformer(BaseEstimator, TransformerMixin, OnnxOperatorMixin):
"""
Calls :epkg:`onnxruntime` inference following :epkg:`scikit-learn` API
so that it can be included in a :epkg:`scikit-learn` pipeline.
Parameters
----------
onnx_bytes : bytes
output_name: string
requested output name or None to request all and
have method *transform* to store all of them in a dataframe
enforce_float32 : boolean
:epkg:`onnxruntime` only supports *float32*,
:epkg:`scikit-learn` usually uses double floats, this parameter
ensures that every array of double floats is converted into
single floats
"""
def __init__(self, onnx_bytes, output_name=None, enforce_float32=True):
BaseEstimator.__init__(self)
TransformerMixin.__init__(self)
self.onnx_bytes = onnx_bytes
self.output_name = output_name
self.enforce_float32 = enforce_float32
if not isinstance(onnx_bytes, bytes):
raise TypeError("onnx_bytes must be bytes to be pickled.")
def __repr__(self):
"""
usual
"""
ob = self.onnx_bytes
if len(ob) > 20:
ob = ob[:10] + b"..." + ob[-10:]
return "{0}(onnx_bytes=b'{1}', output_name={2}, enforce_float32={3})".format(
self.__class__.__name__, ob, self.output_name, enforce_float32)
def fit(self, X=None, y=None, **fit_params):
"""
Loads the :epkg:`ONNX` model.
Parameters
----------
X : unused
y : unused
Returns
-------
self
"""
self.onnxrt_ = InferenceSession(self.onnx_bytes)
self.inputs_ = [_.name for _ in self.onnxrt_.get_inputs()]
return self
def _check_arrays(self, inputs):
"""
Ensures that double floats are converted into single floats
if *enforce_float32* is True or raises an exception.
"""
for k in inputs:
v = inputs[k]
if isinstance(v, numpy.ndarray):
if v.dtype == numpy.float64:
if self.enforce_float32:
inputs[k] = v.astype(numpy.float32)
else:
raise TypeError(
"onnxunruntime only supports floats. Input '{0}' "
"should be converted.".format(k))
def transform(self, X, y=None, **inputs):
"""
Runs the predictions. If *X* is a dataframe,
the function assumes every columns is a separate input,
otherwise, *X* is considered as a first input and *inputs*
can be used to specify extra inputs.
Parameters
----------
X : iterable, data to process (or first input if several expected)
y : unused
inputs: :epkg:`ONNX` graph support multiple inputs,
each column of a dataframe is converted into as many inputs if
*X* is a dataframe, otherwise, *X* is considered as the first input
and *inputs* can be used to specify the other ones
Returns
-------
:epkg:`DataFrame`
"""
if not hasattr(self, "onnxrt_"):
raise AttributeError(
"Transform OnnxTransformer must be fit first.")
rt_inputs = {}
if isinstance(X, pandas.DataFrame):
for c in X.columns:
rt_inputs[c] = X[c]
elif isinstance(X, numpy.ndarray):
rt_inputs[self.inputs_[0]] = X
elif isinstance(X, dict) and len(inputs) == 0:
for k, v in X.items():
rt_inputs[k] = v
elif isinstance(X, list):
if len(self.inputs_) == 1:
rt_inputs[self.inputs_[0]] = numpy.array(X)
else:
for i in range(len(self.inputs_)):
rt_inputs[self.inputs_[i]] = [row[i] for row in X]
for k, v in inputs.items():
rt_inputs[k] = v
names = [self.output_name] if self.output_name else None
self._check_arrays(rt_inputs)
outputs = self.onnxrt_.run(names, rt_inputs)
if self.output_name or len(outputs) == 1:
if isinstance(outputs[0], list):
return pandas.DataFrame(outputs[0])
else:
return outputs[0]
else:
names = self.output_name if self.output_name else [
o.name for o in self.onnxrt_.get_outputs()
]
return pandas.DataFrame({k: v for k, v in zip(names, outputs)})
def fit_transform(self, X, y=None, **inputs):
"""
Loads the *ONNX* model and runs the predictions.
Parameters
----------
X : iterable, data to process (or first input if several expected)
y : unused
inputs: :epkg:`ONNX` graph support multiple inputs,
each column of a dataframe is converted into as many inputs if
*X* is a dataframe, otherwise, *X* is considered as the first input
and *inputs* can be used to specify the other ones
Returns
-------
:epkg:`DataFrame`
"""
return self.fit(X, y=y, **inputs).transform(X, y)
@staticmethod
def enumerate_create(onnx_bytes, output_names=None, enforce_float32=True):
"""
Creates multiple *OnnxTransformer*,
one for each requested intermediate node.
onnx_bytes : bytes
output_names: string
requested output names or None to request all and
have method *transform* to store all of them in a dataframe
enforce_float32 : boolean
:epkg:`onnxruntime` only supports *float32*,
:epkg:`scikit-learn` usually uses double floats, this parameter
ensures that every array of double floats is converted into
single floats
:return: iterator on OnnxTransformer *('output name', OnnxTransformer)*
"""
selected = None if output_names is None else set(output_names)
model = load_onnx_model(onnx_bytes)
for out in enumerate_model_node_outputs(model):
m = select_model_inputs_outputs(model, out)
if selected is None or out in selected:
tr = OnnxTransformer(m.SerializeToString(),
enforce_float32=enforce_float32)
yield out, tr
def onnx_parser(self, inputs=None):
"""
Returns a parser for this model.
"""
if inputs:
self.parsed_inputs_ = inputs
def parser():
return [o.name for o in self.onnxrt_.get_outputs()]
return parser
def onnx_shape_calculator(self):
def shape_calculator(operator):
cout = self.onnxrt_.get_outputs()
if len(operator.outputs) != len(cout):
raise RuntimeError("Mismatched number of outputs: {} != {}."
"".format(len(operator.outputs), len(cout)))
for out in operator.outputs:
shape = out.type.shape
typ = guess_type(out.type)
out.type = typ(shape=shape)
return shape_calculator
def onnx_converter(self):
"""
Returns a converter for this model.
If not overloaded, it fetches the converter
mapped to the first *scikit-learn* parent
it can find.
"""
inputs = getattr(self, "parsed_inputs_", None)
if inputs is None:
inputs = []
for inp in self.onnxrt_.get_inputs():
shape = inp.type.shape
typ = guess_type(inp.type)
inputs.append((inp.name, typ(shape)))
if outputs is None:
outputs = [out.name for out in self.onnxrt_.get_outputs()]
def copy_inout(inout):
shape = [s.dim_value for s in inout.type.tensor_type.shape.dim]
value_info = helper.make_tensor_value_info(
clean_name(inout.name), inout.type.tensor_type.elem_type,
shape)
return value_info
def clean_variable_name(name, scope):
return scope.get_unique_variable_name(naame)
def clean_operator_name(name, scope):
return scope.get_unique_operator_name(naame)
def clean_initializer_name(name, scope):
return scope.get_unique_variable_name(naame)
def converter(scope, operator, container):
graph = model_onnx.graph
inputs = [copy_inout(o) for o in graph.input]
outputs = [copy_inout(o) for o in graph.output]
for node in graph.node:
n = helper.make_node(
node.op_type, [clean_variable_name(o) for o in node.input],
[clean_variable_name(o) for o in node.output])
n.attribute.extend(node.attribute) # pylint: disable=E1101
container.nodes.append(n)
inits = []
for o in graph.initializer:
tensor = TensorProto()
tensor.data_type = o.data_type
tensor.name = clean_initializer_name(o.name)
tensor.raw_data = o.raw_data
tensor.dims.extend(o.dims) # pylint: disable=E1101
container.initializers.append(tensor)
return converter
print(list(sorted(data["data"])))
print(data["data"]['skl'])
##################################
# The input data is the following:
if good:
print(data['data']['data'])
########################################
# Let's compare predictions.
if good:
model_skl = data["data"]['skl']
model_onnx = InferenceSession(data["data"]['ort_onnx'].SerializeToString())
input_name = model_onnx.get_inputs()[0].name
def ort_predict_proba(sess, input, input_name):
res = model_onnx.run(None, {input_name: input.astype(numpy.float32)})[1]
return pandas.DataFrame(res).values
if good:
pred_skl = [
model_skl.predict_proba(input[0]) for input in data['data']['data']
]
pred_onnx = [
ort_predict_proba(model_onnx, input[0], input_name)
for input in data['data']['data']
]
else:
print("'%s' already downloaded" % name)
model_name = "squeezenet1.1-7.onnx"
url_name = ("https://github.com/onnx/models/raw/master/vision/"
"classification/squeezenet/model")
url_name += "/" + model_name
download_file(url_name, model_name, 100000)
################################################
# Loading the ONNX file and use it on one image.
sess = InferenceSession(model_name)
for inp in sess.get_inputs():
print(inp)
#####################################
# The model expects a series of images of size
# `[3, 224, 224]`.
##########################################
# Classifying an image
# ++++++++++++++++++++
url = ("https://upload.wikimedia.org/wikipedia/commons/d/d2/"
"East_Coker_elm%2C_2.jpg")
img = "East_Coker_elm.jpg"
download_file(url, img, 100000)
class DatasetsOrtBenchPerfTest(BenchPerfTest):
def __init__(self, model, dataset, norm):
BenchPerfTest.__init__(self)
self.model_name = model
self.dataset_name = dataset
self.datas = common_datasets[dataset]
skl_model = get_model(model)
if norm:
if 'NB' in model:
self.model = make_pipeline(MinMaxScaler(), skl_model)
else:
self.model = make_pipeline(StandardScaler(), skl_model)
else:
self.model = skl_model
self.model.fit(self.datas[0], self.datas[2])
self.data_test = self.datas[1]
if '-cdist' in model:
options = {id(skl_model): {'optim': 'cdist'}}
else:
options = None
self.onx = to_onnx(self.model,
self.datas[0].astype(numpy.float32),
options=options,
target_opset=__max_supported_opset__)
self.onx.ir_version = get_ir_version(__max_supported_opset__)
logger = getLogger("skl2onnx")
logger.propagate = False
logger.disabled = True
self.ort = InferenceSession(self.onx.SerializeToString())
self.oinf = OnnxInference(self.onx, runtime='python')
self.oinfc = OnnxInference(self.onx, runtime='python_compiled')
self.output_name = self.oinf.output_names[-1]
self.input_name = self.ort.get_inputs()[0].name
self.model_info = analyze_model(self.model)
def fcts(self, **kwargs):
def predict_ort(X, model=self.ort, namei=self.input_name):
try:
return model.run(None, {namei: X})[1]
except Exception as e:
return None
def predict_skl(X, model=self.model):
return model.predict(X)
def predict_pyrt(X,
model=self.oinf,
namei=self.input_name,
nameo=self.output_name):
return model.run({namei: X})[nameo]
def predict_pyrtc(X,
model=self.oinfc,
namei=self.input_name,
nameo=self.output_name):
return model.run({namei: X})[nameo]
return [{
'lib': 'ort',
'fct': predict_ort
}, {
'lib': 'skl',
'fct': predict_skl
}, {
'lib': 'pyrt',
'fct': predict_pyrt
}, {
'lib': 'pyrtc',
'fct': predict_pyrtc
}]
def data(self, N=10, dim=-1, **kwargs): # pylint: disable=W0221
if dim != -1:
raise ValueError("dim must be -1 as it is fixed.")
nbs = numpy.random.randint(0, self.data_test.shape[0] - 1, N)
res = self.data_test[nbs, :].astype(numpy.float32)
return (res, )
def validate(self, results, **kwargs):
nb = 5
if len(results) != nb * 4: # skl, ort, pyrt, pyrtc
raise RuntimeError("Expected only 3 results not {0}.".format(
len(results)))
res = {}
for idt, fct, vals in results:
if idt not in res:
res[idt] = {}
if isinstance(vals, list):
vals = pandas.DataFrame(vals).values
lib = fct['lib']
res[idt][lib] = vals
if len(res) != nb:
raise RuntimeError("Expected only 2 results not {0}.".format(
len(results)))
final = {}
for diff_name in ['ort', 'pyrt', 'pyrtc']:
diffs = []
for i in range(0, nb):
r = res[i]
if diff_name not in r or r[diff_name] is None:
continue
bas = numpy.squeeze(r['skl'])
onn = numpy.squeeze(r[diff_name].squeeze())
if bas.shape != onn.shape:
raise AssertionError(
"Shape mismatch {} != {} params={}".format(
bas.shape, onn.shape, results[0][0]))
diff = numpy.max(numpy.abs(onn - bas))
diffs.append(diff)
if len(diffs) > 0:
final.update({
'diff_%s' % diff_name: sum(diffs) / nb,
'upper_diff_%s' % diff_name: max(diffs),
'lower_diff_%s' % diff_name: min(diffs)
})
for k, v in self.model_info.items():
final['fit_' + k] = v
return final
class InferenceSession: # pylint: disable=E0102
"""
Wrappers around InferenceSession from :epkg:`onnxruntime`.
:param onnx_bytes: onnx bytes
:param session_options: session options
:param log_severity_level: change the logging level
:param device: device, a string `cpu`, `cuda`, `cuda:0`...
"""
def __init__(self,
onnx_bytes,
sess_options=None,
log_severity_level=4,
device=None):
if InferenceSession is None:
raise ImportError( # pragma: no cover
"onnxruntime is not available.")
self.log_severity_level = log_severity_level
if device is None:
self.device = get_ort_device('cpu')
else:
self.device = get_ort_device(device)
self.providers = device_to_providers(self.device)
set_default_logger_severity(3)
if sess_options is None:
self.so = SessionOptions()
self.so.log_severity_level = log_severity_level
self.sess = OrtInferenceSession(onnx_bytes,
sess_options=self.so,
providers=self.providers)
else:
self.so = sess_options
self.sess = OrtInferenceSession(onnx_bytes,
sess_options=sess_options,
providers=self.providers)
self.ro = RunOptions()
self.ro.log_severity_level = log_severity_level
self.ro.log_verbosity_level = log_severity_level
self.output_names = [o.name for o in self.get_outputs()]
def run(self, output_names, input_feed, run_options=None):
"""
Executes the ONNX graph.
:param output_names: None for all, a name for a specific output
:param input_feed: dictionary of inputs
:param run_options: None or RunOptions
:return: array
"""
if any(map(lambda v: isinstance(v, C_OrtValue), input_feed.values())):
return self.sess._sess.run_with_ort_values(input_feed,
self.output_names,
run_options or self.ro)
return self.sess.run(output_names, input_feed, run_options or self.ro)
def get_inputs(self):
"Returns input types."
return self.sess.get_inputs()
def get_outputs(self):
"Returns output types."
return self.sess.get_outputs()
def end_profiling(self):
"Ends profiling."
return self.sess.end_profiling()