def fit(self, X=None, y=None, **fit_params):
"""
Loads the :epkg:`ONNX` model.
:param X: unused
:param y: unused
:param fit_params: additional parameter (unused)
:return: self
"""
from ..onnxrt.optim.onnx_helper import change_input_first_dimension
onx = onnx.load(BytesIO(self.onnx_bytes))
output_names = set(
o.name for o in onx.graph.output) # pylint: disable=E1101
updated = False
if (self.output_name is not None and
self.output_name not in output_names):
# The model refers to intermediate outputs.
onx = select_model_inputs_outputs(
onx, outputs=[self.output_name])
updated = True
if self.change_batch_size is not None:
onx = change_input_first_dimension(
onx, self.change_batch_size)
updated = True
onnx_bytes = (
onx.SerializeToString() if updated else self.onnx_bytes)
self.onnxrt_ = OnnxInference(onnx_bytes, runtime=self.runtime)
self.inputs_ = self.onnxrt_.input_names
return self
def test_model_knn_regressor_radius(self):
model, X = self._fit_model(RadiusNeighborsRegressor())
model_onnx = convert_sklearn(model, "KNN regressor",
[("input", FloatTensorType([None, 4]))],
target_opset=TARGET_OPSET,
options={id(model): {'optim': 'cdist'}})
sess = InferenceSession(model_onnx.SerializeToString())
got = sess.run(None, {'input': X.astype(numpy.float32)})[0]
exp = model.predict(X.astype(numpy.float32))
if any(numpy.isnan(got.ravel())):
# The model is unexpectedly producing nan values
# not on all platforms.
rows = ['--EXP--', str(exp), '--GOT--', str(got),
'--EVERY-OUTPUT--']
for out in enumerate_model_node_outputs(
model_onnx, add_node=False):
onx = select_model_inputs_outputs(model_onnx, out)
sess = InferenceSession(onx.SerializeToString())
res = sess.run(
None, {'input': X.astype(numpy.float32)})
rows.append('--{}--'.format(out))
rows.append(str(res))
if (StrictVersion(onnxruntime.__version__) <
StrictVersion("1.4.0")):
return
raise AssertionError('\n'.join(rows))
assert_almost_equal(exp.ravel(), got.ravel(), decimal=3)
def _display_intermediate_steps(model_onnx, inputs):
import onnxruntime
print("[_display_intermediate_steps] BEGIN")
if isinstance(model_onnx, str):
import onnx
model_onnx = onnx.load(model_onnx)
for name, node in enumerate_model_initializers(model_onnx, add_node=True):
print("INIT: {} - {}".format(name, _guess_type(node)))
for out, node in enumerate_model_node_outputs(model_onnx, add_node=True):
print('-')
print("OUTPUT: {} from {}".format(out, node.name))
step = select_model_inputs_outputs(model_onnx, out)
try:
step_sess = onnxruntime.InferenceSession(step.SerializeToString())
except Exception as e:
raise RuntimeError("Unable to load ONNX model with onnxruntime. "
"Last added node is:\n{}".format(node)) from e
for o in step_sess.get_inputs():
print("IN :", o)
for o in step_sess.get_outputs():
print("OUT: ", o)
if inputs:
res = step_sess.run(inputs)
print(res)
print("[_display_intermediate_steps] END")
def _display_intermediate_steps(model_onnx, inputs, disable_optimisation):
import onnxruntime
print("[_display_intermediate_steps] BEGIN")
if isinstance(model_onnx, str):
import onnx
model_onnx = onnx.load(model_onnx)
for name, node in enumerate_model_initializers(model_onnx, add_node=True):
print("INIT: {} - {}".format(name, _guess_type(node)))
for out, node in enumerate_model_node_outputs(model_onnx, add_node=True):
print('-')
print("OUTPUT: {} from {}".format(out, node.name))
step = select_model_inputs_outputs(model_onnx, out)
if (disable_optimisation
and hasattr(onnxruntime, 'GraphOptimizationLevel')):
opts = onnxruntime.SessionOptions()
opts.graph_optimization_level = (
onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL)
else:
opts = None
try:
step_sess = onnxruntime.InferenceSession(step.SerializeToString(),
sess_options=opts)
except Exception as e:
raise RuntimeError("Unable to load ONNX model with onnxruntime. "
"Last added node is:\n{}".format(node)) from e
for o in step_sess.get_inputs():
print("IN :", o)
for o in step_sess.get_outputs():
print("OUT: ", o)
if inputs:
res = step_sess.run(inputs)
print(res)
print("[_display_intermediate_steps] END")
def save_model(model, feature_names, model_path, label_text="label"):
p, extension = os.path.splitext(model_path)
model.feature_names = feature_names
pickle_path = p + ".pkl"
if extension == ".pmml":
try:
from sklearn2pmml import sklearn2pmml, PMMLPipeline
except ImportError:
raise ImportError(
"You need to install `sklearn2pmml` to store models in pmml format"
)
pipeline = PMMLPipeline([("model", model)])
pipeline.target_field = label_text
pipeline.active_fields = np.array(feature_names)
sklearn2pmml(pipeline, model_path)
elif extension == ".onnx":
try:
from skl2onnx import convert_sklearn
from skl2onnx.common.data_types import FloatTensorType
from skl2onnx.helpers.onnx_helper import select_model_inputs_outputs
from onnx.onnx_pb import StringStringEntryProto
except ImportError:
raise ImportError(
"You need to install `skl2onnx` to store models in onnx format"
)
onnx = convert_sklearn(
model,
name=label_text,
initial_types=[("input", FloatTensorType((None, len(feature_names))))],
doc_string="Model created by aict-tools to estimate {}".format(label_text),
)
# this makes sure we only get the scores and that they are numpy arrays and not
# a list of dicts.
# must come before setting metadata as it clears the metadata_props
if hasattr(model, "predict_proba"):
onnx = select_model_inputs_outputs(onnx, ["probabilities"])
metadata = dict(
model_author="aict-tools",
aict_tools_version=__version__,
feature_names=",".join(feature_names),
model_type="classifier" if is_classifier(model) else "regressor",
)
for key, value in metadata.items():
onnx.metadata_props.append(StringStringEntryProto(key=key, value=value))
with open(model_path, "wb") as f:
f.write(onnx.SerializeToString())
else:
pickle_path = model_path
# Always store the pickle dump,just in case
joblib.dump(model, pickle_path, compress=4)
def print_specific_output(model_path,
input_tensor,
output_name,
print_tensor=False):
model_onnx = load_onnx_model(model_path)
num_onnx = select_model_inputs_outputs(model_onnx, output_name)
save_onnx_model(num_onnx, "remove_temp.onnx")
sess = rt.InferenceSession("remove_temp.onnx")
out_tensor = sess.run(None, input_tensor)
print("name", output_name, "shape", out_tensor[0].shape)
if print_tensor:
print(out_tensor[0])
def test_onnx_helper_load_save_init_meta(self):
model = make_pipeline(Binarizer(), OneHotEncoder(sparse=False),
StandardScaler())
X = numpy.array([[0.1, 1.1], [0.2, 2.2], [0.4, 2.2], [0.2, 2.4]])
model.fit(X)
model_onnx = convert_sklearn(model, "pipe3",
[("input", FloatTensorType([None, 2]))])
meta = {'pA': 'one', 'pB': 'two'}
onnx.helper.set_model_props(model_onnx, meta)
new_model = select_model_inputs_outputs(model_onnx, "variable")
vals = {p.key: p.value for p in new_model.metadata_props}
assert vals == meta
def test_model_knn_regressor2_1_radius(self):
model, X = self._fit_model_simple(
RadiusNeighborsRegressor(algorithm="brute"), n_targets=2)
X = X[:-1]
model_onnx = convert_sklearn(
model,
"KNN regressor", [("input", FloatTensorType([None, X.shape[1]]))],
target_opset=TARGET_OPSET)
self.assertIsNotNone(model_onnx)
sess = InferenceSession(model_onnx.SerializeToString())
got = sess.run(None, {'input': X.astype(numpy.float32)})[0]
exp = model.predict(X.astype(numpy.float32))
if any(numpy.isnan(got.ravel())):
# The model is unexpectedly producing nan values
# not on all platforms.
# It happens when two matrices are multiplied,
# one is (2, 20, 20), second is (20, 20)
# and contains only 0 or 1 values.
# The output contains nan values on the first row
# but not on the second one.
rows = [
'--EXP--',
str(exp), '--GOT--',
str(got), '--EVERY-OUTPUT--'
]
for out in enumerate_model_node_outputs(model_onnx,
add_node=False):
onx = select_model_inputs_outputs(model_onnx, out)
sess = InferenceSession(onx.SerializeToString())
res = sess.run(None, {'input': X.astype(numpy.float32)})
rows.append('--{}--'.format(out))
rows.append(str(res))
if (onnxruntime.__version__.startswith('1.4.')
or onnxruntime.__version__.startswith('1.5.')):
# TODO: investigate the regression in onnxruntime 1.4
# One broadcasted multiplication unexpectedly produces nan.
whole = '\n'.join(rows)
if "[ nan" in whole:
warnings.warn(whole)
return
raise AssertionError(whole)
if (onnxruntime.__version__.startswith('1.3.')
and sys.platform == 'win32'):
# Same error but different line number for further
# investigation.
raise AssertionError(whole)
raise AssertionError('\n'.join(rows))
assert_almost_equal(exp, got, decimal=5)
def _modify_model_add_outputs_nodes(self, model_dir):
old_onnx_model = onnx.load(self.args.model_path)
utils.print_info_log("load model success")
for index, node in enumerate(old_onnx_model.graph.node):
if not node.name:
node.name = node.op_type + "_" + str(index)
outputs_name = [
name for name in enumerate_model_node_outputs(old_onnx_model)
]
new_onnx_model = select_model_inputs_outputs(old_onnx_model,
outputs_name)
new_onnx_model_path = os.path.join(
model_dir, "new_" + os.path.basename(self.args.model_path))
save_onnx_model(new_onnx_model, new_onnx_model_path)
utils.print_info_log("modify model outputs success")
return old_onnx_model, new_onnx_model_path
def test_onnx_helper_load_save(self):
model = make_pipeline(StandardScaler(), Binarizer(threshold=0.5))
X = numpy.array([[0.1, 1.1], [0.2, 2.2]])
model.fit(X)
model_onnx = convert_sklearn(model, "binarizer",
[("input", FloatTensorType([1, 2]))])
filename = "temp_onnx_helper_load_save.onnx"
save_onnx_model(model_onnx, filename)
model = load_onnx_model(filename)
new_model = select_model_inputs_outputs(model, "variable")
assert new_model.graph is not None
tr1 = self.get_model(model)
tr2 = self.get_model(new_model)
X = X.astype(numpy.float32)
X1 = tr1(X)
X2 = tr2(X)
assert X1.shape == (2, 2)
assert X2.shape == (2, 2)
def test_onnx_helper_load_save_init(self):
model = make_pipeline(Binarizer(), OneHotEncoder(sparse=False),
StandardScaler())
X = numpy.array([[0.1, 1.1], [0.2, 2.2], [0.4, 2.2], [0.2, 2.4]])
model.fit(X)
model_onnx = convert_sklearn(model, "pipe3",
[("input", FloatTensorType([None, 2]))])
filename = "temp_onnx_helper_load_save.onnx"
save_onnx_model(model_onnx, filename)
model = load_onnx_model(filename)
new_model = select_model_inputs_outputs(model, "variable")
assert new_model.graph is not None
tr1 = self.get_model(model)
tr2 = self.get_model(new_model)
X = X.astype(numpy.float32)
X1 = tr1(X)
X2 = tr2(X)
assert X1.shape == (4, 2)
assert X2.shape == (4, 2)
def test_onnx_helper_load_save(self):
model = make_pipeline(StandardScaler(), Binarizer(threshold=0.5))
X = numpy.array([[0.1, 1.1], [0.2, 2.2]])
model.fit(X)
model_onnx = convert_sklearn(model, 'binarizer',
[('input', FloatTensorType([1, 2]))])
filename = "temp_onnx_helper_load_save.onnx"
save_onnx_model(model_onnx, filename)
model = load_onnx_model(filename)
list(enumerate_model_node_outputs(model))
new_model = select_model_inputs_outputs(model, 'variable')
self.assertTrue(new_model.graph is not None)
tr1 = self.get_model(model)
tr2 = self.get_model(new_model)
X = X.astype(numpy.float32)
X1 = tr1(X)
X2 = tr2(X)
self.assertEqual(X1.shape, (2, 2))
self.assertEqual(X2.shape, (2, 2))
def test_onnx_helper_load_save_init(self):
model = make_pipeline(Binarizer(), OneHotEncoder(sparse=False),
StandardScaler())
X = numpy.array([[0.1, 1.1], [0.2, 2.2], [0.4, 2.2], [0.2, 2.4]])
model.fit(X)
model_onnx = convert_sklearn(model, 'pipe3',
[('input', FloatTensorType([1, 2]))])
filename = "temp_onnx_helper_load_save.onnx"
save_onnx_model(model_onnx, filename)
model = load_onnx_model(filename)
list(enumerate_model_node_outputs(model))
new_model = select_model_inputs_outputs(model, 'variable')
self.assertTrue(new_model.graph is not None) # pylint: disable=E1101
tr1 = self.get_model(model)
tr2 = self.get_model(new_model)
X = X.astype(numpy.float32)
X1 = tr1(X)
X2 = tr2(X)
self.assertEqual(X1.shape, (4, 2))
self.assertEqual(X2.shape, (4, 2))
def test_model_knn_regressor2_1_radius(self):
model, X = self._fit_model_simple(
RadiusNeighborsRegressor(algorithm="brute"), n_targets=2)
model_onnx = convert_sklearn(
model,
"KNN regressor", [("input", FloatTensorType([None, X.shape[1]]))],
target_opset=TARGET_OPSET)
self.assertIsNotNone(model_onnx)
sess = InferenceSession(model_onnx.SerializeToString())
got = sess.run(None, {'input': X.astype(numpy.float32)})[0]
exp = model.predict(X.astype(numpy.float32))
if any(numpy.isnan(got.ravel())):
# The model is unexpectedly producing nan values
# not on all platforms.
# It happens when two matrices are multiplied,
# one is (2, 20, 20), second is (20, 20)
# and contains only 0 or 1 values.
# The output contains nan values on the first row
# but not on the second one.
rows = [
'--EXP--',
str(exp), '--GOT--',
str(got), '--EVERY-OUTPUT--'
]
for out in enumerate_model_node_outputs(model_onnx,
add_node=False):
onx = select_model_inputs_outputs(model_onnx, out)
sess = InferenceSession(onx.SerializeToString())
res = sess.run(None, {'input': X.astype(numpy.float32)})
rows.append('--{}--'.format(out))
rows.append(str(res))
if (StrictVersion(onnxruntime.__version__) <
StrictVersion("1.4.0")):
return
raise AssertionError('\n'.join(rows))
assert_almost_equal(exp, got, decimal=5)
def enumerate_create(onnx_bytes, output_names=None, enforce_float32=True):
"""
Creates multiple *OnnxTransformer*,
one for each requested intermediate node.
onnx_bytes : bytes
output_names: string
requested output names or None to request all and
have method *transform* to store all of them in a dataframe
enforce_float32 : boolean
:epkg:`onnxruntime` only supports *float32*,
:epkg:`scikit-learn` usually uses double floats, this parameter
ensures that every array of double floats is converted into
single floats
:return: iterator on OnnxTransformer *('output name', OnnxTransformer)*
"""
selected = None if output_names is None else set(output_names)
model = load_onnx_model(onnx_bytes)
for out in enumerate_model_node_outputs(model):
m = select_model_inputs_outputs(model, out)
if selected is None or out in selected:
tr = OnnxTransformer(m.SerializeToString(),
enforce_float32=enforce_float32)
yield out, tr
# We need to modifies the *ONNX* before it is given to *onnxruntime*.
# Let's see first the list of intermediate output.
model_onnx = load_onnx_model("pipeline_titanic.onnx")
for out in enumerate_model_node_outputs(model_onnx):
print(out)
################################
# Not that easy to tell which one is what as the *ONNX*
# has more operators than the original *scikit-learn* pipelines.
# The graph at :ref:`l-plot-complex-pipeline-graph`
# helps up to find the outputs of both numerical
# and textual pipeline: *variable1*, *variable2*.
# Let's look into the numerical pipeline first.
num_onnx = select_model_inputs_outputs(model_onnx, 'variable1')
save_onnx_model(num_onnx, "pipeline_titanic_numerical.onnx")
################################
# Let's compute the numerical features.
sess = rt.InferenceSession("pipeline_titanic_numerical.onnx")
numX = sess.run(None, inputs)
print("numerical features", numX[0][:1])
###########################################
# We do the same for the textual features.
print(model_onnx)
text_onnx = select_model_inputs_outputs(model_onnx, 'variable2')
save_onnx_model(text_onnx, "pipeline_titanic_textual.onnx")
inputs.append(onnx.load_tensor(input_file))
"""
# For some models this string has to match the input to the model
inputDict = {"image": inputs[0]}
outputs_list = []
for out in enumerate_model_node_outputs(model_onnx):
outputs_list.append(out)
print(outputs_list)
for idx, out in enumerate(outputs_list):
name = str(idx) + "_" + out
dataset = "test_data_set_0"
os.mkdir(name)
os.mkdir(name + "/" + dataset)
modelPath = name + "/" + name + ".onnx"
model_output = select_model_inputs_outputs(model_onnx, out)
save_onnx_model(model_output, modelPath)
sess = rt.InferenceSession(modelPath)
numX = sess.run(None, inputDict)
print()
print("Generating idx=", idx)
print(out)
print(numX)
# hardcoded for 1 output
numpy_to_pb(out, numX[0], name + "/" + dataset + "/" + "output_0.pb")
numpy_to_pb(out, inputs[0], name + "/" + dataset + "/" + "input_0.pb")
def benchmark_tflite(url,
dest,
onnx_name,
opset,
imgs,
verbose=True,
threshold=1e-3,
names=None):
"""
Runs a simple benchmark with a tflite model.
Goes through every steps (download, convert).
Skips them if already done.
"""
if url.startswith('http'):
tname = download_tflite(url, dest)
if verbose:
print("Created %r." % tname)
else:
tname = url
# Converts the model.
if verbose:
print("Convert model in %r." % dest)
convert_tflite(tname, onnx_name, opset)
if verbose:
print("Created %r." % onnx_name)
# Benchmarks both models.
ort = onnxruntime.InferenceSession(onnx_name)
if verbose:
print("ONNX inputs:")
for a in ort.get_inputs():
print(" {}: {}, {}".format(a.name, a.type, a.shape))
print("ONNX outputs:")
for a in ort.get_outputs():
print(" {}: {}, {}".format(a.name, a.type, a.shape))
# onnxruntime
input_name = ort.get_inputs()[0].name
fct_ort = lambda img: ort.run(None, {input_name: img})[0]
results_ort, duration_ort = measure_time(fct_ort, imgs)
if verbose:
print("ORT", len(imgs), duration_ort)
# tflite
import tensorflow as tf
interpreter = tf.lite.Interpreter(tname)
#help(interpreter)
input_details = interpreter.get_input_details()
index_in = input_details[0]['index']
output_details = interpreter.get_output_details()
index_out = output_details[0]['index']
interpreter.allocate_tensors()
def call_tflite(inp):
interpreter.set_tensor(index_in, inp)
interpreter.invoke()
scores = interpreter.get_tensor(index_out)
return scores
# check intermediate results
if names is not None:
from skl2onnx.helpers.onnx_helper import select_model_inputs_outputs
import onnx
with open(onnx_name, "rb") as f:
model_onnx = onnx.load(f)
interpreter_details = tf.lite.Interpreter(
tname, experimental_preserve_all_tensors=True)
input_details = interpreter_details.get_input_details()
index_in = input_details[0]['index']
interpreter_details.allocate_tensors()
interpreter_details.set_tensor(index_in, imgs[0])
interpreter_details.invoke()
details = interpreter_details.get_tensor_details()
inputs = {input_name: imgs[0]}
names_index = {}
for tt in details:
names_index[tt['name']] = (tt['index'], tt['quantization'],
tt['quantization_parameters'])
num_results = []
for name_tfl, name_ort in names:
index = names_index[name_tfl]
tfl_value = interpreter_details.get_tensor(index[0])
new_name = onnx_name + ".%s.onnx" % name_ort.replace(
":", "_").replace(";", "_").replace("/", "_")
if not os.path.exists(new_name):
print('[create onnx model for %r, %r.' % (name_tfl, name_ort))
new_model = select_model_inputs_outputs(model_onnx,
outputs=[name_ort])
with open(new_name, "wb") as f:
f.write(new_model.SerializeToString())
ort_inter = onnxruntime.InferenceSession(new_name)
result = ort_inter.run(None, inputs)[0]
diff = numpy.abs(tfl_value.ravel().astype(numpy.float64) -
result.ravel().astype(numpy.float64)).max()
num_results.append("diff=%f names=(%r,%r) " %
(diff, name_tfl, name_ort))
print("*** diff=%f names=(%r,%r) " % (diff, name_tfl, name_ort))
print(" TFL:", tfl_value.dtype, tfl_value.shape,
tfl_value.min(), tfl_value.max())
print(" ORT:", result.dtype, result.shape, result.min(),
result.max())
print("\n".join(num_results))
results_tfl, duration_tfl = measure_time(call_tflite, imgs)
if verbose:
print("TFL", len(imgs), duration_tfl)
mean_ort = sum(duration_ort) / len(duration_ort)
mean_tfl = sum(duration_tfl) / len(duration_tfl)
print("ratio ORT=%r / TF=%r = %r" %
(mean_ort, mean_tfl, mean_ort / mean_tfl))
# checks discrepencies
res = call_tflite(imgs[0])
res_ort = fct_ort(imgs[0])
if isinstance(res, dict):
if len(res) != 1:
raise NotImplementedError(
"TF output contains more than one output: %r." % res)
output_name = ort.get_outputs()[0].name
if output_name not in res:
raise AssertionError("Unable to find output %r in %r." %
(output_name, list(sorted(res))))
res = res[output_name]
check_discrepencies(res_ort, res, threshold)
return duration_ort, duration_tf
print("outputs")
pprint.pprint(res)
#######################################
# The major draw back of this solution is increase the prediction
# time as onnxruntime copies the constants for every prediction.
# It is possible either to store those constant in a separate ONNX graph
# or to removes them.
#
# Select outputs
# ++++++++++++++
#
# Next function removes unneeded outputs from a model,
# not only the constants. Next model only keeps the probabilities.
simple_onx = select_model_inputs_outputs(new_onx, ['probabilities'])
sess = InferenceSession(simple_onx.SerializeToString(),
providers=['CPUExecutionProvider'])
print("output names:", [o.name for o in sess.get_outputs()])
res = sess.run(None, {'X': X_test[:2]})
print("outputs")
pprint.pprint(res)
# Function *select_model_inputs_outputs* add also promote an intermediate
# result to an output.
#
#####################################
# This example only uses ONNX graph in memory and never saves or loads a
# model. This can be done by using the following snippets of code.
#
