def get_io_numpy_type_map(
ort_session: InferenceSession) -> Dict[str, numpy.dtype]:
"""Create a mapping from input/output name to numpy data type"""
name_to_numpy_type = {}
for input in ort_session.get_inputs():
name_to_numpy_type[input.name] = TypeHelper.ort_type_to_numpy_type(
input.type)
for output in ort_session.get_outputs():
name_to_numpy_type[
output.name] = TypeHelper.ort_type_to_numpy_type(output.type)
return name_to_numpy_type
def format_results(sess: rt.InferenceSession, data: list) -> dict:
input_name = sess.get_inputs()[0].name
input_shape = sess.get_inputs()[0].shape
label_name = sess.get_outputs()[0].name
return {
'input_name': input_name,
'input_shape': input_shape[1:],
'output_name': label_name,
'prediction': data,
'createdAt': datetime.now(timezone.utc).astimezone().isoformat()
}
def onnx_predict(sess: InferenceSession, x: np.ndarray):
"""
use ONNX runtime session to predict result
:param sess: ONNX runtime session
:param x: input ndarray
:return: predicted result
"""
x = x.reshape((-1, 1, 28, 28))
input_name = sess.get_inputs()[0].name
label_name = sess.get_outputs()[0].name
pred = sess.run([label_name], {input_name: x.astype("float32")})[0]
return np.argmax(pred, axis=1)[0]
def _test_lgbm(self, X, model, extra_config={}):
# Create ONNX-ML model
onnx_ml_model = convert_model(
model, 'lgbm-onnxml', [("input", FloatTensorType([X.shape[0], X.shape[1]]))]
)[0]
# Create ONNX model
onnx_model = convert_model(
model, 'lgbm-onnx', [("input", FloatTensorType([X.shape[0], X.shape[1]]))], without_onnx_ml=True
)[0]
try:
from onnxruntime import InferenceSession
except ImportError:
# onnxruntime not installed (python 2.7)
return
# Get the predictions for the ONNX-ML model
session = InferenceSession(onnx_ml_model.SerializeToString())
output_names = [session.get_outputs()[i].name for i in range(len(session.get_outputs()))]
onnx_ml_pred = [[] for i in range(len(output_names))]
inputs = {session.get_inputs()[0].name: X}
pred = session.run(output_names, inputs)
for i in range(len(output_names)):
if output_names[i] == "label":
onnx_ml_pred[1] = pred[i]
else:
onnx_ml_pred[0] = pred[i]
# Get the predictions for the ONNX model
session = InferenceSession(onnx_model.SerializeToString())
onnx_pred = [[] for i in range(len(output_names))]
pred = session.run(output_names, inputs)
for i in range(len(output_names)):
if output_names[i] == "label":
onnx_pred[1] = pred[i]
else:
onnx_pred[0] = pred[i]
return onnx_ml_pred, onnx_pred, output_names
def fcts_model(X, y, max_depth, n_estimators, n_jobs):
"RandomForestClassifier."
rf = RandomForestRegressor(max_depth=max_depth,
n_estimators=n_estimators,
n_jobs=n_jobs)
rf.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = convert_sklearn(rf, initial_types=initial_types)
f = BytesIO()
f.write(onx.SerializeToString())
content = f.getvalue()
sess = InferenceSession(content)
outputs = [o.name for o in sess.get_outputs()]
if False:
import treelite.sklearn
import treelite_runtime
try:
lite = treelite.sklearn.import_model(rf)
name = "lite{}.dll".format(id(rf))
lite.export_lib(
toolchain='msvc' if sys.platform == "win32" else "gcc",
libpath=name,
verbose=False)
lite_predictor = treelite_runtime.Predictor(name, verbose=False)
except (treelite.util.TreeliteError, PermissionError,
UnicodeDecodeError):
lite_predictor = None
def predict_skl_predict(X, model=rf):
return rf.predict(X)
def predict_onnxrt_predict(X, sess=sess):
return sess.run(outputs[:1], {'X': X})[0]
def predict_treelite_predict(X, sess=sess):
return numpy.array(
lite_predictor.predict(
treelite_runtime.Batch.from_npy2d(X.astype(np.float32))))
return {
'predict': (
predict_skl_predict,
predict_onnxrt_predict,
None,
)
}
def fcts_model(X, y, n_jobs):
"LinearRegression."
model = LinearRegression(n_jobs=n_jobs)
model.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = to_onnx(model,
initial_types=initial_types,
black_op={'LinearRegressor'})
sess = InferenceSession(onx.SerializeToString(),
providers=['CPUExecutionProvider'])
outputs = [o.name for o in sess.get_outputs()]
oinf = OnnxInference(onx, runtime="python")
bind = SessionIOBinding(sess._sess)
# ort_device = C_OrtDevice.cpu()
ort_device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(),
0)
def predict_skl_predict(X, model=model):
return model.predict(X)
def predict_onnxrt_predict(X, sess=sess):
return sess.run(outputs[:1], {'X': X})[0]
def predict_onnx_inference(X, oinf=oinf):
return oinf.run({'X': X})["variable"]
def predict_onnxrt_predict_bind(X,
sess=sess,
bind=bind,
ort_device=ort_device):
if X.__array_interface__['strides'] is not None:
raise RuntimeError("onnxruntime only supports contiguous arrays.")
bind.bind_input('X', ort_device, X.dtype, X.shape,
X.__array_interface__['data'][0])
bind.bind_output('variable', ort_device)
sess._sess.run_with_iobinding(bind, None)
ortvalues = bind.get_outputs()
return ortvalues[0].numpy()
return {
'predict': {
'skl': predict_skl_predict,
'ort': predict_onnxrt_predict,
'numpy': predict_onnx_inference,
'ort-bind': predict_onnxrt_predict_bind
}
}
def test_local_outlier_factor_double(self):
lof = LocalOutlierFactor(n_neighbors=2, novelty=True)
data = np.array([[-1.1, -1.2], [0.3, 0.2], [0.5, 0.4], [100., 99.]],
dtype=np.float64)
model = lof.fit(data)
model_onnx = to_onnx(model, data, target_opset=TARGET_OPSET)
sess = InferenceSession(model_onnx.SerializeToString())
names = [o.name for o in sess.get_outputs()]
self.assertEqual(names, ['label', 'scores'])
got = sess.run(None, {'X': data})
self.assertEqual(len(got), 2)
expected_label = lof.predict(data)
expected_decif = lof.decision_function(data)
assert_almost_equal(expected_label, got[0].ravel())
assert_almost_equal(expected_decif, got[1].ravel())
def c_test_model(self, model):
model, X = fit_classification_model(
model, 3, n_features=4, label_string=False)
model_onnx = convert_sklearn(
model, "multi-class ridge classifier",
[("input", FloatTensorType([None, X.shape[1]]))],
options={id(model): {'zipmap': 'columns'}},
target_opset=TARGET_OPSET)
self.assertIsNotNone(model_onnx)
sess = InferenceSession(model_onnx.SerializeToString())
names = [_.name for _ in sess.get_outputs()]
self.assertEqual(['output_label', 'i0', 'i1', 'i2'], names)
xt = X[:10].astype(np.float32)
got = sess.run(None, {'input': xt})
prob = model.predict_proba(xt)
for i in range(prob.shape[1]):
assert_almost_equal(prob[:, i], got[i+1])
def test_local_outlier_factor_rnd(self):
lof = LocalOutlierFactor(n_neighbors=2, novelty=True)
rs = np.random.RandomState(0)
data = rs.randn(100, 4).astype(np.float32)
data[-1, 2:] = 99.
data[-2, :2] = -99.
model = lof.fit(data)
model_onnx = to_onnx(model, data, target_opset=TARGET_OPSET)
sess = InferenceSession(model_onnx.SerializeToString())
names = [o.name for o in sess.get_outputs()]
self.assertEqual(names, ['label', 'scores'])
got = sess.run(None, {'X': data})
self.assertEqual(len(got), 2)
expected_label = lof.predict(data)
expected_decif = lof.decision_function(data)
assert_almost_equal(expected_label, got[0].ravel())
assert_almost_equal(expected_decif, got[1].ravel(), decimal=5)
def test_model_logistic_regression_binary_class(self):
model, X = fit_classification_model(
linear_model.LogisticRegression(max_iter=100), 2)
model_onnx = convert_sklearn(
model, "logistic regression",
[("input", FloatTensorType([None, X.shape[1]]))],
target_opset=TARGET_OPSET)
self.assertIsNotNone(model_onnx)
dump_data_and_model(
X, model, model_onnx,
basename="SklearnLogitisticRegressionBinary")
if pv.Version(ort_version) >= pv.Version("1.0.0"):
sess = InferenceSession(model_onnx.SerializeToString())
out = sess.get_outputs()
lb = out[0].type
sh = out[0].shape
self.assertEqual(str(lb), "tensor(int64)")
self.assertEqual(sh, [None])
def fcts_model(X, y, max_depth, n_estimators, n_jobs):
"RandomForestClassifier."
rf = RandomForestClassifier(max_depth=max_depth,
n_estimators=n_estimators,
n_jobs=n_jobs)
rf.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = convert_sklearn(rf,
initial_types=initial_types,
options={id(rf): {
'zipmap': False
}})
sess = InferenceSession(onx.SerializeToString())
outputs = [o.name for o in sess.get_outputs()]
oinf = OnnxInference(onx, runtime="python")
oinf.sequence_[0].ops_._init(numpy.float32, 1)
name = outputs[1]
oinf2 = OnnxInference(onx, runtime="python")
oinf2.sequence_[0].ops_._init(numpy.float32, 2)
oinf3 = OnnxInference(onx, runtime="python")
oinf3.sequence_[0].ops_._init(numpy.float32, 3)
def predict_skl_predict(X, model=rf):
return rf.predict_proba(X)
def predict_onnxrt_predict(X, sess=sess):
return sess.run(outputs[:1], {'X': X})[0]
def predict_onnx_inference(X, oinf=oinf):
return oinf.run({'X': X})[name]
def predict_onnx_inference2(X, oinf2=oinf2):
return oinf2.run({'X': X})[name]
def predict_onnx_inference3(X, oinf3=oinf3):
return oinf3.run({'X': X})[name]
return {
'predict':
(predict_skl_predict, predict_onnxrt_predict, predict_onnx_inference,
predict_onnx_inference2, predict_onnx_inference3)
}
def test_isolation_forest_score_samples(self):
isol = IsolationForest(n_estimators=3, random_state=0)
data = np.array([[-1.1, -1.2], [0.3, 0.2], [0.5, 0.4], [100., 99.]],
dtype=np.float32)
model = isol.fit(data)
model_onnx = to_onnx(model,
data,
target_opset=TARGET_OPSET,
options={'score_samples': True})
sess = InferenceSession(model_onnx.SerializeToString())
names = [o.name for o in sess.get_outputs()]
self.assertEqual(names, ['label', 'scores', 'score_samples'])
got = sess.run(None, {'X': data})
self.assertEqual(len(got), 3)
expected_label = isol.predict(data)
expected_decif = isol.decision_function(data)
expected_score = isol.score_samples(data)
assert_almost_equal(expected_label, got[0].ravel())
assert_almost_equal(expected_decif, got[1].ravel())
assert_almost_equal(expected_score, got[2].ravel())
def test_kmeans(self):
model = KMeans()
X, y = make_regression(n_features=4, random_state=42)
model.fit(X, y)
initial_types = [('input', FloatTensorType((None, X.shape[1])))]
with self.assertRaises(RuntimeError):
convert_sklearn(model, initial_types=initial_types,
final_types=[('output4', None)])
with self.assertRaises(RuntimeError):
convert_sklearn(model, initial_types=initial_types,
final_types=[('dup1', None), ('dup1', None)],
target_opset=TARGET_OPSET)
model_onnx = convert_sklearn(
model, initial_types=initial_types,
final_types=[('output4', None), ('output5', None)],
target_opset=TARGET_OPSET)
assert model_onnx is not None
sess = InferenceSession(model_onnx.SerializeToString())
assert sess.get_outputs()[0].name == 'output4'
assert sess.get_outputs()[1].name == 'output5'
def fcts_model(X, y, fit_intercept):
"LinearRegression."
rf = LinearRegression(fit_intercept=fit_intercept)
rf.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = convert_sklearn(rf, initial_types=initial_types)
f = BytesIO()
f.write(onx.SerializeToString())
content = f.getvalue()
sess = InferenceSession(content)
outputs = [o.name for o in sess.get_outputs()]
def predict_skl_predict(X, model=rf):
return rf.predict(X)
def predict_onnxrt_predict(X, sess=sess):
return sess.run(outputs[:1], {'X': X})[0]
return {'predict': (predict_skl_predict, predict_onnxrt_predict)}
class OnnxModelLoader:
def __init__(self, onnx_path: str) -> None:
"""
Class for loading ONNX models to inference on CPU. CPU inference is very effective using onnxruntime.
:param onnx_path: path to ONNX model file (*.onnx file).
"""
self.sess = InferenceSession(onnx_path,
providers=['CPUExecutionProvider'])
self.input_name = [x.name for x in self.sess.get_inputs()][0]
self.output_names = [x.name for x in self.sess.get_outputs()]
def inference(self, inputs: np.ndarray) -> List[np.ndarray]:
"""
Run inference.
:param inputs: list of arguments, order must match names in input_names.
:return: list of outputs.
"""
return self.sess.run(self.output_names,
input_feed={self.input_name: inputs})
def test_lightgbm_booster_multi_classifier(self):
X = [[0, 1], [1, 1], [2, 0], [1, 2], [-1, 2], [1, -2]]
X = numpy.array(X, dtype=numpy.float32)
y = [0, 1, 0, 1, 2, 2]
data = lightgbm.Dataset(X, label=y)
model = lightgbm.train({'boosting_type': 'gbdt', 'objective': 'multiclass',
'n_estimators': 3, 'min_child_samples': 1, 'num_class': 3},
data)
model_onnx, prefix = convert_model(model, 'tree-based classifier',
[('input', FloatTensorType([None, 2]))])
dump_data_and_model(X, model, model_onnx,
allow_failure="StrictVersion(onnx.__version__) < StrictVersion('1.3.0')",
basename=prefix + "BoosterBin" + model.__class__.__name__)
try:
from onnxruntime import InferenceSession
except ImportError:
# onnxruntime not installed (python 2.7)
return
sess = InferenceSession(model_onnx.SerializeToString())
out = sess.get_outputs()
names = [o.name for o in out]
assert names == ['label', 'probabilities']
def benchmark(name,
onx,
fct_numpy,
*args,
dims=(1, 10, 100, 200, 500, 1000, 2000, 10000)):
sess = InferenceSession(onx.SerializeToString())
device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0)
names = [i.name for i in sess.get_inputs()]
out_names = [o.name for o in sess.get_outputs()]
if len(names) != len(args):
raise RuntimeError(f"Size mismatch {len(names)} != {len(args)}.")
rows = []
for dim in tqdm(dims):
new_args = [reshape(a, dim) for a in args]
ortvalues = [
C_OrtValue.ortvalue_from_numpy(a, device) for a in new_args
]
ms = measure_time(lambda: fct_numpy(*new_args), repeat=50, number=100)
ms.update(dict(name=name, impl='numpy', dim=dim))
rows.append(ms)
inps = {n: a for n, a in zip(names, new_args)}
ms = measure_time(lambda: sess.run(None, inps))
ms.update(dict(name=name, impl='sess', dim=dim))
rows.append(ms)
bind = SessionIOBinding(sess._sess)
ms = measure_time(lambda: bind_and_run(sess._sess, bind, names,
ortvalues, out_names, device))
ms.update(dict(name=name, impl='bind_run', dim=dim))
rows.append(ms)
ms = measure_time(lambda: nobind_just_run(sess._sess, bind))
ms.update(dict(name=name, impl='run', dim=dim))
rows.append(ms)
return rows
def fcts_model(X, y, max_depth, n_estimators):
"RandomForestClassifier."
rf = RandomForestClassifier(max_depth=max_depth, n_estimators=n_estimators)
rf.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = convert_sklearn(rf, initial_types=initial_types)
f = BytesIO()
f.write(onx.SerializeToString())
content = f.getvalue()
sess = InferenceSession(content)
outputs = [o.name for o in sess.get_outputs()]
def predict_skl_predict(X, model=rf):
return rf.predict(X)
def predict_skl_predict_proba(X, model=rf):
return rf.predict_proba(X)
def predict_onnxrt_predict(X, sess=sess):
return numpy.array(sess.run(outputs[:1], {'X': X.astype(np.float32)}))
def predict_onnxrt_predict_proba(X, sess=sess):
res = sess.run(outputs[1:], {'X': X.astype(np.float32)})[0]
# do not use DataFrame to convert the output into array,
# it takes too much time
out = numpy.empty((len(res), len(res[0])), dtype=numpy.float32)
for i, row in enumerate(res):
for k, v in row.items():
out[i, k] = v
return out
return {
'predict': (predict_skl_predict, predict_onnxrt_predict),
'predict_proba':
(predict_skl_predict_proba, predict_onnxrt_predict_proba)
}
def test_local_outlier_factor_metric(self):
for metric in ['cityblock', 'euclidean', 'manhattan', 'sqeuclidean']:
with self.subTest(metric=metric):
lof = LocalOutlierFactor(n_neighbors=2,
novelty=True,
metric=metric)
data = np.array(
[[-1.1, -1.2], [0.3, 0.2], [0.5, 0.4], [100., 99.]],
dtype=np.float32)
model = lof.fit(data)
model_onnx = to_onnx(model, data, target_opset=TARGET_OPSET)
data = data.copy()
data[:, 0] += 0.1
sess = InferenceSession(model_onnx.SerializeToString())
names = [o.name for o in sess.get_outputs()]
self.assertEqual(names, ['label', 'scores'])
got = sess.run(None, {'X': data})
self.assertEqual(len(got), 2)
expected_label = lof.predict(data)
expected_decif = lof.decision_function(data)
assert_almost_equal(expected_label, got[0].ravel())
assert_almost_equal(expected_decif, got[1].ravel(), decimal=4)
def fcts_model(X, y, max_depth, n_estimators, n_jobs):
"RandomForestClassifier."
rf = RandomForestClassifier(max_depth=max_depth,
n_estimators=n_estimators,
n_jobs=n_jobs)
rf.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = convert_sklearn(rf,
initial_types=initial_types,
options={RandomForestClassifier: {
'zipmap': False
}})
f = BytesIO()
f.write(onx.SerializeToString())
content = f.getvalue()
sess = InferenceSession(content)
outputs = [o.name for o in sess.get_outputs()]
def predict_skl_predict(X, model=rf):
return rf.predict(X)
def predict_skl_predict_proba(X, model=rf):
return rf.predict_proba(X)
def predict_onnxrt_predict(X, sess=sess):
return sess.run(outputs[:1], {'X': X})[0]
def predict_onnxrt_predict_proba(X, sess=sess):
return sess.run(outputs[1:], {'X': X})[0]
return {
'predict': (predict_skl_predict, predict_onnxrt_predict),
'predict_proba':
(predict_skl_predict_proba, predict_onnxrt_predict_proba)
}
def fcts_model(X, y, n_jobs):
"LinearRegression."
model = LinearRegression(n_jobs=n_jobs)
model.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = convert_sklearn(model, initial_types=initial_types)
sess = InferenceSession(onx.SerializeToString())
outputs = [o.name for o in sess.get_outputs()]
oinf = OnnxInference(onx, runtime="python")
def predict_skl_predict(X, model=model):
return model.predict(X)
def predict_onnxrt_predict(X, sess=sess):
return sess.run(outputs[:1], {'X': X})[0]
def predict_onnx_inference(X, oinf=oinf):
return oinf.run({'X': X})["variable"]
return {
'predict':
(predict_skl_predict, predict_onnxrt_predict, predict_onnx_inference)
}
clrrf = RandomForestClassifier(n_estimators=2, max_depth=2)
clrrf.fit(X_train, y_train)
clrrf.predict(X_test[:2])
paths, n_nodes_ptr = clrrf.decision_path(X_test[:2])
print(paths.todense())
model_def = to_onnx(clrrf, X_train.astype(numpy.float32),
options={id(clrrf): {'decision_path': True,
'zipmap': False}})
sess = InferenceSession(model_def.SerializeToString())
##########################################
# The model produces 3 outputs.
print([o.name for o in sess.get_outputs()])
##########################################
# Let's display the last one.
res = sess.run(None, {'X': X_test[:2].astype(numpy.float32)})
print(res[-1])
############################################################
# List of available options
# +++++++++++++++++++++++++
#
# Options are registered for every converted to detect any
# supported options while running the conversion.
def predict(sess: rt.InferenceSession, data: list):
input_name = sess.get_inputs()[0].name
label_name = sess.get_outputs()[0].name
return sess.run([label_name], { input_name: data })[0]
class OnnxTransformer(BaseEstimator, TransformerMixin, OnnxOperatorMixin):
"""
Calls :epkg:`onnxruntime` inference following :epkg:`scikit-learn` API
so that it can be included in a :epkg:`scikit-learn` pipeline.
Parameters
----------
onnx_bytes : bytes
output_name: string
requested output name or None to request all and
have method *transform* to store all of them in a dataframe
enforce_float32 : boolean
:epkg:`onnxruntime` only supports *float32*,
:epkg:`scikit-learn` usually uses double floats, this parameter
ensures that every array of double floats is converted into
single floats
"""
def __init__(self, onnx_bytes, output_name=None, enforce_float32=True):
BaseEstimator.__init__(self)
TransformerMixin.__init__(self)
self.onnx_bytes = onnx_bytes
self.output_name = output_name
self.enforce_float32 = enforce_float32
if not isinstance(onnx_bytes, bytes):
raise TypeError("onnx_bytes must be bytes to be pickled.")
def __repr__(self):
"""
usual
"""
ob = self.onnx_bytes
if len(ob) > 20:
ob = ob[:10] + b"..." + ob[-10:]
return "{0}(onnx_bytes=b'{1}', output_name={2}, enforce_float32={3})".format(
self.__class__.__name__, ob, self.output_name, enforce_float32)
def fit(self, X=None, y=None, **fit_params):
"""
Loads the :epkg:`ONNX` model.
Parameters
----------
X : unused
y : unused
Returns
-------
self
"""
self.onnxrt_ = InferenceSession(self.onnx_bytes)
self.inputs_ = [_.name for _ in self.onnxrt_.get_inputs()]
return self
def _check_arrays(self, inputs):
"""
Ensures that double floats are converted into single floats
if *enforce_float32* is True or raises an exception.
"""
for k in inputs:
v = inputs[k]
if isinstance(v, numpy.ndarray):
if v.dtype == numpy.float64:
if self.enforce_float32:
inputs[k] = v.astype(numpy.float32)
else:
raise TypeError(
"onnxunruntime only supports floats. Input '{0}' "
"should be converted.".format(k))
def transform(self, X, y=None, **inputs):
"""
Runs the predictions. If *X* is a dataframe,
the function assumes every columns is a separate input,
otherwise, *X* is considered as a first input and *inputs*
can be used to specify extra inputs.
Parameters
----------
X : iterable, data to process (or first input if several expected)
y : unused
inputs: :epkg:`ONNX` graph support multiple inputs,
each column of a dataframe is converted into as many inputs if
*X* is a dataframe, otherwise, *X* is considered as the first input
and *inputs* can be used to specify the other ones
Returns
-------
:epkg:`DataFrame`
"""
if not hasattr(self, "onnxrt_"):
raise AttributeError(
"Transform OnnxTransformer must be fit first.")
rt_inputs = {}
if isinstance(X, pandas.DataFrame):
for c in X.columns:
rt_inputs[c] = X[c]
elif isinstance(X, numpy.ndarray):
rt_inputs[self.inputs_[0]] = X
elif isinstance(X, dict) and len(inputs) == 0:
for k, v in X.items():
rt_inputs[k] = v
elif isinstance(X, list):
if len(self.inputs_) == 1:
rt_inputs[self.inputs_[0]] = numpy.array(X)
else:
for i in range(len(self.inputs_)):
rt_inputs[self.inputs_[i]] = [row[i] for row in X]
for k, v in inputs.items():
rt_inputs[k] = v
names = [self.output_name] if self.output_name else None
self._check_arrays(rt_inputs)
outputs = self.onnxrt_.run(names, rt_inputs)
if self.output_name or len(outputs) == 1:
if isinstance(outputs[0], list):
return pandas.DataFrame(outputs[0])
else:
return outputs[0]
else:
names = self.output_name if self.output_name else [
o.name for o in self.onnxrt_.get_outputs()
]
return pandas.DataFrame({k: v for k, v in zip(names, outputs)})
def fit_transform(self, X, y=None, **inputs):
"""
Loads the *ONNX* model and runs the predictions.
Parameters
----------
X : iterable, data to process (or first input if several expected)
y : unused
inputs: :epkg:`ONNX` graph support multiple inputs,
each column of a dataframe is converted into as many inputs if
*X* is a dataframe, otherwise, *X* is considered as the first input
and *inputs* can be used to specify the other ones
Returns
-------
:epkg:`DataFrame`
"""
return self.fit(X, y=y, **inputs).transform(X, y)
@staticmethod
def enumerate_create(onnx_bytes, output_names=None, enforce_float32=True):
"""
Creates multiple *OnnxTransformer*,
one for each requested intermediate node.
onnx_bytes : bytes
output_names: string
requested output names or None to request all and
have method *transform* to store all of them in a dataframe
enforce_float32 : boolean
:epkg:`onnxruntime` only supports *float32*,
:epkg:`scikit-learn` usually uses double floats, this parameter
ensures that every array of double floats is converted into
single floats
:return: iterator on OnnxTransformer *('output name', OnnxTransformer)*
"""
selected = None if output_names is None else set(output_names)
model = load_onnx_model(onnx_bytes)
for out in enumerate_model_node_outputs(model):
m = select_model_inputs_outputs(model, out)
if selected is None or out in selected:
tr = OnnxTransformer(m.SerializeToString(),
enforce_float32=enforce_float32)
yield out, tr
def onnx_parser(self, inputs=None):
"""
Returns a parser for this model.
"""
if inputs:
self.parsed_inputs_ = inputs
def parser():
return [o.name for o in self.onnxrt_.get_outputs()]
return parser
def onnx_shape_calculator(self):
def shape_calculator(operator):
cout = self.onnxrt_.get_outputs()
if len(operator.outputs) != len(cout):
raise RuntimeError("Mismatched number of outputs: {} != {}."
"".format(len(operator.outputs), len(cout)))
for out in operator.outputs:
shape = out.type.shape
typ = guess_type(out.type)
out.type = typ(shape=shape)
return shape_calculator
def onnx_converter(self):
"""
Returns a converter for this model.
If not overloaded, it fetches the converter
mapped to the first *scikit-learn* parent
it can find.
"""
inputs = getattr(self, "parsed_inputs_", None)
if inputs is None:
inputs = []
for inp in self.onnxrt_.get_inputs():
shape = inp.type.shape
typ = guess_type(inp.type)
inputs.append((inp.name, typ(shape)))
if outputs is None:
outputs = [out.name for out in self.onnxrt_.get_outputs()]
def copy_inout(inout):
shape = [s.dim_value for s in inout.type.tensor_type.shape.dim]
value_info = helper.make_tensor_value_info(
clean_name(inout.name), inout.type.tensor_type.elem_type,
shape)
return value_info
def clean_variable_name(name, scope):
return scope.get_unique_variable_name(naame)
def clean_operator_name(name, scope):
return scope.get_unique_operator_name(naame)
def clean_initializer_name(name, scope):
return scope.get_unique_variable_name(naame)
def converter(scope, operator, container):
graph = model_onnx.graph
inputs = [copy_inout(o) for o in graph.input]
outputs = [copy_inout(o) for o in graph.output]
for node in graph.node:
n = helper.make_node(
node.op_type, [clean_variable_name(o) for o in node.input],
[clean_variable_name(o) for o in node.output])
n.attribute.extend(node.attribute) # pylint: disable=E1101
container.nodes.append(n)
inits = []
for o in graph.initializer:
tensor = TensorProto()
tensor.data_type = o.data_type
tensor.name = clean_initializer_name(o.name)
tensor.raw_data = o.raw_data
tensor.dims.extend(o.dims) # pylint: disable=E1101
container.initializers.append(tensor)
return converter
so.optimized_model_filepath = os.path.split(filename)[-1] + ".optimized.onnx"
sess = InferenceSession(onx.SerializeToString(), so, providers=[provider])
bind = SessionIOBinding(sess._sess)
print("graph_optimization_level:", so.graph_optimization_level)
#####################################
# Creates random data
feed = random_feed(sess, batch)
#####################################
# moving the data on CPU or GPU
feed_ort_value = OrderedDict(
(name, (C_OrtValue.ortvalue_from_numpy(v, ort_device), v.dtype))
for name, v in feed.items())
outputs = [o.name for o in sess.get_outputs()]
#######################################
# A function which calls the API for any device.
def run_with_iobinding(sess, bind, ort_device, feed_ort_value, outputs):
for name, (value, dtype) in feed_ort_value.items():
bind.bind_input(name, ort_device, dtype, value.shape(),
value.data_ptr())
for out in outputs:
bind.bind_output(out, ort_device)
sess._sess.run_with_iobinding(bind, None)
ortvalues = bind.get_outputs()
return [o.numpy() for o in ortvalues]
# Add training parameter
# ++++++++++++++++++++++
#
new_onx = add_output_initializer(
onx, ['C', 'l1_ratio'],
[numpy.array([model.C]),
numpy.array([model.l1_ratio])])
########################################
# Inference
# +++++++++
sess = InferenceSession(new_onx.SerializeToString(),
providers=['CPUExecutionProvider'])
print("output names:", [o.name for o in sess.get_outputs()])
res = sess.run(None, {'X': X_test[:2]})
print("outputs")
pprint.pprint(res)
#######################################
# The major draw back of this solution is increase the prediction
# time as onnxruntime copies the constants for every prediction.
# It is possible either to store those constant in a separate ONNX graph
# or to removes them.
#
# Select outputs
# ++++++++++++++
#
# Next function removes unneeded outputs from a model,
# not only the constants. Next model only keeps the probabilities.
class InferenceSession: # pylint: disable=E0102
"""
Wrappers around InferenceSession from :epkg:`onnxruntime`.
:param onnx_bytes: onnx bytes
:param session_options: session options
:param log_severity_level: change the logging level
:param device: device, a string `cpu`, `cuda`, `cuda:0`...
"""
def __init__(self,
onnx_bytes,
sess_options=None,
log_severity_level=4,
device=None):
if InferenceSession is None:
raise ImportError( # pragma: no cover
"onnxruntime is not available.")
self.log_severity_level = log_severity_level
if device is None:
self.device = get_ort_device('cpu')
else:
self.device = get_ort_device(device)
self.providers = device_to_providers(self.device)
set_default_logger_severity(3)
if sess_options is None:
self.so = SessionOptions()
self.so.log_severity_level = log_severity_level
self.sess = OrtInferenceSession(onnx_bytes,
sess_options=self.so,
providers=self.providers)
else:
self.so = sess_options
self.sess = OrtInferenceSession(onnx_bytes,
sess_options=sess_options,
providers=self.providers)
self.ro = RunOptions()
self.ro.log_severity_level = log_severity_level
self.ro.log_verbosity_level = log_severity_level
self.output_names = [o.name for o in self.get_outputs()]
def run(self, output_names, input_feed, run_options=None):
"""
Executes the ONNX graph.
:param output_names: None for all, a name for a specific output
:param input_feed: dictionary of inputs
:param run_options: None or RunOptions
:return: array
"""
if any(map(lambda v: isinstance(v, C_OrtValue), input_feed.values())):
return self.sess._sess.run_with_ort_values(input_feed,
self.output_names,
run_options or self.ro)
return self.sess.run(output_names, input_feed, run_options or self.ro)
def get_inputs(self):
"Returns input types."
return self.sess.get_inputs()
def get_outputs(self):
"Returns output types."
return self.sess.get_outputs()
def end_profiling(self):
"Ends profiling."
return self.sess.end_profiling()
def _create_onnx_graphs(self):
"""
Creates forward and backward ONNX graph.
The new class has the following attributes:
* `__doc__`: doc string
* `__module__`: module name (this file)
* `_run_options`: see :epkg:`RunOptions`
* `_sess`: :epkg:`InferenceSession` with the original graph
* `_sess_eval`: :epkg:`InferenceSession` on the graph
with weights as inputs
* `_training_agent`: :epkg:`TrainingAgent`
* `_cache`: :epkg:`OrtValueCache`
* `_logger`: logger
* `_input_names`: input names
* `_debug`: use debug mode
* `_grad_input_names`: gradient input names
* `_output_names`: output names
* `_weights_to_train`: names of the weights to train
Training attributes
* `_bw_fetches_names`: bw_fetches_names,
* `_fw_outputs_device_info`: fw_outputs_device_info,
* `_bw_outputs_device_info`: bw_outputs_device_info,
* `_fw_no_grad_output_device_info`: fw_no_grad_output_device_info,
* `_graph_info`: graph_info}
Additional attributes added if *keep_model* is True:
* `_trained_onnx`: ONNX graph for the gradient
* `_optimized_pre_grad_model`: evaluation ONNX graph taking
weights as inputs
* `_graph_builder`: :epkg:`OrtModuleGraphBuilder`
"""
logger = self._logger
if logger is not None:
logger.info("[OrtGradientForwardBackward] create training onnx")
logger.info("[OrtGradientForwardBackward] input_names=%r",
self.input_names)
logger.info("[OrtGradientForwardBackward] output_names=%r",
self.output_names)
logger.info("[OrtGradientForwardBackward] weights_to_train=%r",
self.weights_to_train)
builder = OrtModuleGraphBuilder()
if logger is not None:
cf = self.graph_builder_config.graph_transformer_config
cfp = cf.propagate_cast_ops_config
logger.info("[OrtGradientForwardBackward] "
"OrtModuleGraphBuilder.initialize")
logger.info(
"[OrtGradientForwardBackward] graph_builder_config=%s",
OrtGradientForwardBackward._repr_helper_(
self.graph_builder_config, indent=4))
logger.info(
"[OrtGradientForwardBackward] graph_builder_config."
"graph_transformer_config=%s",
OrtGradientForwardBackward._repr_helper_(cf, indent=4))
logger.info(
"[OrtGradientForwardBackward] graph_builder_config."
"graph_transformer_config.propagate_cast_ops_config=%s",
OrtGradientForwardBackward._repr_helper_(cfp, indent=4))
builder.initialize(self.onnx_model.SerializeToString(),
self.graph_builder_config)
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] OrtModuleGraphBuilder.build")
builder.build()
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] OrtModuleGraphBuilder.get_model")
train_onnx_model_serialized = builder.get_model()
optimized_pre_grad_model = builder.get_inference_optimized_model()
graph_info = builder.get_graph_info()
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] graph_info=%s",
OrtGradientForwardBackward._repr_helper_(graph_info, indent=4))
logger.info("[OrtGradientForwardBackward] create TrainSession")
logger.info(
"[OrtGradientForwardBackward] sess_options=%s",
OrtGradientForwardBackward._repr_helper_(self.sess_options,
indent=4))
logger.info("[OrtGradientForwardBackward] providers=%r",
self.providers)
sess = InferenceSession(train_onnx_model_serialized,
sess_options=self.sess_options,
provider_options=self.provider_options,
providers=self.providers)
if logger is not None:
logger.info("[OrtGradientForwardBackward] create InferenceSession")
sess_eval = InferenceSession(optimized_pre_grad_model,
sess_options=self.sess_options,
provider_options=self.provider_options,
providers=self.providers)
if logger is not None:
logger.info("[OrtGradientForwardBackward] create training agent")
grad_input_names = [obj.name for obj in sess.get_inputs()]
bw_fetches_names = [obj.name for obj in sess.get_outputs()]
fw_outputs_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(i),
OrtDevice.default_memory(), self.device_index)
for i in self.providers
]
bw_outputs_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(
self.providers[0]), OrtDevice.default_memory(),
self.device_index) for i in bw_fetches_names
]
fw_no_grad_output_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(
self.providers[0]), OrtDevice.default_memory(),
self.device_index) for i in self.output_names
]
try:
# onnxruntime>=1.12
training_agent = TrainingAgent(sess._sess, grad_input_names,
fw_outputs_device_info,
bw_fetches_names,
bw_outputs_device_info, 0)
except TypeError:
# onnxruntime<=1.11
training_agent = TrainingAgent(sess._sess, grad_input_names,
fw_outputs_device_info,
bw_fetches_names,
bw_outputs_device_info)
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] instantiate dynamic class %r",
self.class_name)
logger.info("[OrtGradientForwardBackward] weights_to_train=%r",
self.weights_to_train)
logger.info("[OrtGradientForwardBackward] grad_input_names=%r",
grad_input_names)
logger.info("[OrtGradientForwardBackward] bw_fetches_names=%r",
bw_fetches_names)
logger.info("[OrtGradientForwardBackward] device_index=%r",
self.device_index)
devices = list(fw_outputs_device_info)
while len(devices) < len(grad_input_names):
devices.append(devices[-1])
trained_onnx = onnx.load(BytesIO(train_onnx_model_serialized))
onnx_loss = onnx.load(BytesIO(optimized_pre_grad_model))
for i, node in enumerate(trained_onnx.graph.node):
if node.name == '':
node.name = "N%d" % i
for i, node in enumerate(onnx_loss.graph.node):
if node.name == '':
node.name = "N%d" % i
kwargs = {
'_run_options': self.run_options,
'_sess': sess,
'_sess_eval': sess_eval,
'_training_agent': training_agent,
'_cache': OrtValueCache(),
'_logger': logger,
'_input_names': self.input_names,
'_grad_input_names': grad_input_names,
'_output_names': self.output_names,
'_bw_fetches_names': bw_fetches_names,
'_fw_outputs_device_info': fw_outputs_device_info,
'_bw_outputs_device_info': bw_outputs_device_info,
'_fw_no_grad_output_device_info': fw_no_grad_output_device_info,
'_weights_to_train': list(sorted(self.weights_to_train)),
'_graph_info': graph_info,
#
'_trained_onnx': trained_onnx,
'_optimized_pre_grad_model': onnx_loss,
'_graph_builder': builder,
'_devices': devices,
'_debug': self.debug
}
graph = kwargs['_trained_onnx'].graph
kwargs.update({
'_onx_inp': [o.name for o in graph.input],
'_onx_out': [o.name for o in graph.output]
})
if len(kwargs['_onx_inp']) != len(kwargs['_onx_out']):
raise RuntimeError( # pragma: no cover
"Gradient input and output are inconsistant: "
"%r != %r" % (kwargs['_onx_inp'], kwargs['_onx_out']))
return kwargs
