def get_io_numpy_type_map( ort_session: InferenceSession) -> Dict[str, numpy.dtype]: """Create a mapping from input/output name to numpy data type""" name_to_numpy_type = {} for input in ort_session.get_inputs(): name_to_numpy_type[input.name] = TypeHelper.ort_type_to_numpy_type( input.type) for output in ort_session.get_outputs(): name_to_numpy_type[ output.name] = TypeHelper.ort_type_to_numpy_type(output.type) return name_to_numpy_type
def format_results(sess: rt.InferenceSession, data: list) -> dict: input_name = sess.get_inputs()[0].name input_shape = sess.get_inputs()[0].shape label_name = sess.get_outputs()[0].name return { 'input_name': input_name, 'input_shape': input_shape[1:], 'output_name': label_name, 'prediction': data, 'createdAt': datetime.now(timezone.utc).astimezone().isoformat() }
def onnx_predict(sess: InferenceSession, x: np.ndarray): """ use ONNX runtime session to predict result :param sess: ONNX runtime session :param x: input ndarray :return: predicted result """ x = x.reshape((-1, 1, 28, 28)) input_name = sess.get_inputs()[0].name label_name = sess.get_outputs()[0].name pred = sess.run([label_name], {input_name: x.astype("float32")})[0] return np.argmax(pred, axis=1)[0]
def _test_lgbm(self, X, model, extra_config={}): # Create ONNX-ML model onnx_ml_model = convert_model( model, 'lgbm-onnxml', [("input", FloatTensorType([X.shape[0], X.shape[1]]))] )[0] # Create ONNX model onnx_model = convert_model( model, 'lgbm-onnx', [("input", FloatTensorType([X.shape[0], X.shape[1]]))], without_onnx_ml=True )[0] try: from onnxruntime import InferenceSession except ImportError: # onnxruntime not installed (python 2.7) return # Get the predictions for the ONNX-ML model session = InferenceSession(onnx_ml_model.SerializeToString()) output_names = [session.get_outputs()[i].name for i in range(len(session.get_outputs()))] onnx_ml_pred = [[] for i in range(len(output_names))] inputs = {session.get_inputs()[0].name: X} pred = session.run(output_names, inputs) for i in range(len(output_names)): if output_names[i] == "label": onnx_ml_pred[1] = pred[i] else: onnx_ml_pred[0] = pred[i] # Get the predictions for the ONNX model session = InferenceSession(onnx_model.SerializeToString()) onnx_pred = [[] for i in range(len(output_names))] pred = session.run(output_names, inputs) for i in range(len(output_names)): if output_names[i] == "label": onnx_pred[1] = pred[i] else: onnx_pred[0] = pred[i] return onnx_ml_pred, onnx_pred, output_names
def fcts_model(X, y, max_depth, n_estimators, n_jobs): "RandomForestClassifier." rf = RandomForestRegressor(max_depth=max_depth, n_estimators=n_estimators, n_jobs=n_jobs) rf.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = convert_sklearn(rf, initial_types=initial_types) f = BytesIO() f.write(onx.SerializeToString()) content = f.getvalue() sess = InferenceSession(content) outputs = [o.name for o in sess.get_outputs()] if False: import treelite.sklearn import treelite_runtime try: lite = treelite.sklearn.import_model(rf) name = "lite{}.dll".format(id(rf)) lite.export_lib( toolchain='msvc' if sys.platform == "win32" else "gcc", libpath=name, verbose=False) lite_predictor = treelite_runtime.Predictor(name, verbose=False) except (treelite.util.TreeliteError, PermissionError, UnicodeDecodeError): lite_predictor = None def predict_skl_predict(X, model=rf): return rf.predict(X) def predict_onnxrt_predict(X, sess=sess): return sess.run(outputs[:1], {'X': X})[0] def predict_treelite_predict(X, sess=sess): return numpy.array( lite_predictor.predict( treelite_runtime.Batch.from_npy2d(X.astype(np.float32)))) return { 'predict': ( predict_skl_predict, predict_onnxrt_predict, None, ) }
def fcts_model(X, y, n_jobs): "LinearRegression." model = LinearRegression(n_jobs=n_jobs) model.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = to_onnx(model, initial_types=initial_types, black_op={'LinearRegressor'}) sess = InferenceSession(onx.SerializeToString(), providers=['CPUExecutionProvider']) outputs = [o.name for o in sess.get_outputs()] oinf = OnnxInference(onx, runtime="python") bind = SessionIOBinding(sess._sess) # ort_device = C_OrtDevice.cpu() ort_device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0) def predict_skl_predict(X, model=model): return model.predict(X) def predict_onnxrt_predict(X, sess=sess): return sess.run(outputs[:1], {'X': X})[0] def predict_onnx_inference(X, oinf=oinf): return oinf.run({'X': X})["variable"] def predict_onnxrt_predict_bind(X, sess=sess, bind=bind, ort_device=ort_device): if X.__array_interface__['strides'] is not None: raise RuntimeError("onnxruntime only supports contiguous arrays.") bind.bind_input('X', ort_device, X.dtype, X.shape, X.__array_interface__['data'][0]) bind.bind_output('variable', ort_device) sess._sess.run_with_iobinding(bind, None) ortvalues = bind.get_outputs() return ortvalues[0].numpy() return { 'predict': { 'skl': predict_skl_predict, 'ort': predict_onnxrt_predict, 'numpy': predict_onnx_inference, 'ort-bind': predict_onnxrt_predict_bind } }
def test_local_outlier_factor_double(self): lof = LocalOutlierFactor(n_neighbors=2, novelty=True) data = np.array([[-1.1, -1.2], [0.3, 0.2], [0.5, 0.4], [100., 99.]], dtype=np.float64) model = lof.fit(data) model_onnx = to_onnx(model, data, target_opset=TARGET_OPSET) sess = InferenceSession(model_onnx.SerializeToString()) names = [o.name for o in sess.get_outputs()] self.assertEqual(names, ['label', 'scores']) got = sess.run(None, {'X': data}) self.assertEqual(len(got), 2) expected_label = lof.predict(data) expected_decif = lof.decision_function(data) assert_almost_equal(expected_label, got[0].ravel()) assert_almost_equal(expected_decif, got[1].ravel())
def c_test_model(self, model): model, X = fit_classification_model( model, 3, n_features=4, label_string=False) model_onnx = convert_sklearn( model, "multi-class ridge classifier", [("input", FloatTensorType([None, X.shape[1]]))], options={id(model): {'zipmap': 'columns'}}, target_opset=TARGET_OPSET) self.assertIsNotNone(model_onnx) sess = InferenceSession(model_onnx.SerializeToString()) names = [_.name for _ in sess.get_outputs()] self.assertEqual(['output_label', 'i0', 'i1', 'i2'], names) xt = X[:10].astype(np.float32) got = sess.run(None, {'input': xt}) prob = model.predict_proba(xt) for i in range(prob.shape[1]): assert_almost_equal(prob[:, i], got[i+1])
def test_local_outlier_factor_rnd(self): lof = LocalOutlierFactor(n_neighbors=2, novelty=True) rs = np.random.RandomState(0) data = rs.randn(100, 4).astype(np.float32) data[-1, 2:] = 99. data[-2, :2] = -99. model = lof.fit(data) model_onnx = to_onnx(model, data, target_opset=TARGET_OPSET) sess = InferenceSession(model_onnx.SerializeToString()) names = [o.name for o in sess.get_outputs()] self.assertEqual(names, ['label', 'scores']) got = sess.run(None, {'X': data}) self.assertEqual(len(got), 2) expected_label = lof.predict(data) expected_decif = lof.decision_function(data) assert_almost_equal(expected_label, got[0].ravel()) assert_almost_equal(expected_decif, got[1].ravel(), decimal=5)
def test_model_logistic_regression_binary_class(self): model, X = fit_classification_model( linear_model.LogisticRegression(max_iter=100), 2) model_onnx = convert_sklearn( model, "logistic regression", [("input", FloatTensorType([None, X.shape[1]]))], target_opset=TARGET_OPSET) self.assertIsNotNone(model_onnx) dump_data_and_model( X, model, model_onnx, basename="SklearnLogitisticRegressionBinary") if pv.Version(ort_version) >= pv.Version("1.0.0"): sess = InferenceSession(model_onnx.SerializeToString()) out = sess.get_outputs() lb = out[0].type sh = out[0].shape self.assertEqual(str(lb), "tensor(int64)") self.assertEqual(sh, [None])
def fcts_model(X, y, max_depth, n_estimators, n_jobs): "RandomForestClassifier." rf = RandomForestClassifier(max_depth=max_depth, n_estimators=n_estimators, n_jobs=n_jobs) rf.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = convert_sklearn(rf, initial_types=initial_types, options={id(rf): { 'zipmap': False }}) sess = InferenceSession(onx.SerializeToString()) outputs = [o.name for o in sess.get_outputs()] oinf = OnnxInference(onx, runtime="python") oinf.sequence_[0].ops_._init(numpy.float32, 1) name = outputs[1] oinf2 = OnnxInference(onx, runtime="python") oinf2.sequence_[0].ops_._init(numpy.float32, 2) oinf3 = OnnxInference(onx, runtime="python") oinf3.sequence_[0].ops_._init(numpy.float32, 3) def predict_skl_predict(X, model=rf): return rf.predict_proba(X) def predict_onnxrt_predict(X, sess=sess): return sess.run(outputs[:1], {'X': X})[0] def predict_onnx_inference(X, oinf=oinf): return oinf.run({'X': X})[name] def predict_onnx_inference2(X, oinf2=oinf2): return oinf2.run({'X': X})[name] def predict_onnx_inference3(X, oinf3=oinf3): return oinf3.run({'X': X})[name] return { 'predict': (predict_skl_predict, predict_onnxrt_predict, predict_onnx_inference, predict_onnx_inference2, predict_onnx_inference3) }
def test_isolation_forest_score_samples(self): isol = IsolationForest(n_estimators=3, random_state=0) data = np.array([[-1.1, -1.2], [0.3, 0.2], [0.5, 0.4], [100., 99.]], dtype=np.float32) model = isol.fit(data) model_onnx = to_onnx(model, data, target_opset=TARGET_OPSET, options={'score_samples': True}) sess = InferenceSession(model_onnx.SerializeToString()) names = [o.name for o in sess.get_outputs()] self.assertEqual(names, ['label', 'scores', 'score_samples']) got = sess.run(None, {'X': data}) self.assertEqual(len(got), 3) expected_label = isol.predict(data) expected_decif = isol.decision_function(data) expected_score = isol.score_samples(data) assert_almost_equal(expected_label, got[0].ravel()) assert_almost_equal(expected_decif, got[1].ravel()) assert_almost_equal(expected_score, got[2].ravel())
def test_kmeans(self): model = KMeans() X, y = make_regression(n_features=4, random_state=42) model.fit(X, y) initial_types = [('input', FloatTensorType((None, X.shape[1])))] with self.assertRaises(RuntimeError): convert_sklearn(model, initial_types=initial_types, final_types=[('output4', None)]) with self.assertRaises(RuntimeError): convert_sklearn(model, initial_types=initial_types, final_types=[('dup1', None), ('dup1', None)], target_opset=TARGET_OPSET) model_onnx = convert_sklearn( model, initial_types=initial_types, final_types=[('output4', None), ('output5', None)], target_opset=TARGET_OPSET) assert model_onnx is not None sess = InferenceSession(model_onnx.SerializeToString()) assert sess.get_outputs()[0].name == 'output4' assert sess.get_outputs()[1].name == 'output5'
def fcts_model(X, y, fit_intercept): "LinearRegression." rf = LinearRegression(fit_intercept=fit_intercept) rf.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = convert_sklearn(rf, initial_types=initial_types) f = BytesIO() f.write(onx.SerializeToString()) content = f.getvalue() sess = InferenceSession(content) outputs = [o.name for o in sess.get_outputs()] def predict_skl_predict(X, model=rf): return rf.predict(X) def predict_onnxrt_predict(X, sess=sess): return sess.run(outputs[:1], {'X': X})[0] return {'predict': (predict_skl_predict, predict_onnxrt_predict)}
class OnnxModelLoader: def __init__(self, onnx_path: str) -> None: """ Class for loading ONNX models to inference on CPU. CPU inference is very effective using onnxruntime. :param onnx_path: path to ONNX model file (*.onnx file). """ self.sess = InferenceSession(onnx_path, providers=['CPUExecutionProvider']) self.input_name = [x.name for x in self.sess.get_inputs()][0] self.output_names = [x.name for x in self.sess.get_outputs()] def inference(self, inputs: np.ndarray) -> List[np.ndarray]: """ Run inference. :param inputs: list of arguments, order must match names in input_names. :return: list of outputs. """ return self.sess.run(self.output_names, input_feed={self.input_name: inputs})
def test_lightgbm_booster_multi_classifier(self): X = [[0, 1], [1, 1], [2, 0], [1, 2], [-1, 2], [1, -2]] X = numpy.array(X, dtype=numpy.float32) y = [0, 1, 0, 1, 2, 2] data = lightgbm.Dataset(X, label=y) model = lightgbm.train({'boosting_type': 'gbdt', 'objective': 'multiclass', 'n_estimators': 3, 'min_child_samples': 1, 'num_class': 3}, data) model_onnx, prefix = convert_model(model, 'tree-based classifier', [('input', FloatTensorType([None, 2]))]) dump_data_and_model(X, model, model_onnx, allow_failure="StrictVersion(onnx.__version__) < StrictVersion('1.3.0')", basename=prefix + "BoosterBin" + model.__class__.__name__) try: from onnxruntime import InferenceSession except ImportError: # onnxruntime not installed (python 2.7) return sess = InferenceSession(model_onnx.SerializeToString()) out = sess.get_outputs() names = [o.name for o in out] assert names == ['label', 'probabilities']
def benchmark(name, onx, fct_numpy, *args, dims=(1, 10, 100, 200, 500, 1000, 2000, 10000)): sess = InferenceSession(onx.SerializeToString()) device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0) names = [i.name for i in sess.get_inputs()] out_names = [o.name for o in sess.get_outputs()] if len(names) != len(args): raise RuntimeError(f"Size mismatch {len(names)} != {len(args)}.") rows = [] for dim in tqdm(dims): new_args = [reshape(a, dim) for a in args] ortvalues = [ C_OrtValue.ortvalue_from_numpy(a, device) for a in new_args ] ms = measure_time(lambda: fct_numpy(*new_args), repeat=50, number=100) ms.update(dict(name=name, impl='numpy', dim=dim)) rows.append(ms) inps = {n: a for n, a in zip(names, new_args)} ms = measure_time(lambda: sess.run(None, inps)) ms.update(dict(name=name, impl='sess', dim=dim)) rows.append(ms) bind = SessionIOBinding(sess._sess) ms = measure_time(lambda: bind_and_run(sess._sess, bind, names, ortvalues, out_names, device)) ms.update(dict(name=name, impl='bind_run', dim=dim)) rows.append(ms) ms = measure_time(lambda: nobind_just_run(sess._sess, bind)) ms.update(dict(name=name, impl='run', dim=dim)) rows.append(ms) return rows
def fcts_model(X, y, max_depth, n_estimators): "RandomForestClassifier." rf = RandomForestClassifier(max_depth=max_depth, n_estimators=n_estimators) rf.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = convert_sklearn(rf, initial_types=initial_types) f = BytesIO() f.write(onx.SerializeToString()) content = f.getvalue() sess = InferenceSession(content) outputs = [o.name for o in sess.get_outputs()] def predict_skl_predict(X, model=rf): return rf.predict(X) def predict_skl_predict_proba(X, model=rf): return rf.predict_proba(X) def predict_onnxrt_predict(X, sess=sess): return numpy.array(sess.run(outputs[:1], {'X': X.astype(np.float32)})) def predict_onnxrt_predict_proba(X, sess=sess): res = sess.run(outputs[1:], {'X': X.astype(np.float32)})[0] # do not use DataFrame to convert the output into array, # it takes too much time out = numpy.empty((len(res), len(res[0])), dtype=numpy.float32) for i, row in enumerate(res): for k, v in row.items(): out[i, k] = v return out return { 'predict': (predict_skl_predict, predict_onnxrt_predict), 'predict_proba': (predict_skl_predict_proba, predict_onnxrt_predict_proba) }
def test_local_outlier_factor_metric(self): for metric in ['cityblock', 'euclidean', 'manhattan', 'sqeuclidean']: with self.subTest(metric=metric): lof = LocalOutlierFactor(n_neighbors=2, novelty=True, metric=metric) data = np.array( [[-1.1, -1.2], [0.3, 0.2], [0.5, 0.4], [100., 99.]], dtype=np.float32) model = lof.fit(data) model_onnx = to_onnx(model, data, target_opset=TARGET_OPSET) data = data.copy() data[:, 0] += 0.1 sess = InferenceSession(model_onnx.SerializeToString()) names = [o.name for o in sess.get_outputs()] self.assertEqual(names, ['label', 'scores']) got = sess.run(None, {'X': data}) self.assertEqual(len(got), 2) expected_label = lof.predict(data) expected_decif = lof.decision_function(data) assert_almost_equal(expected_label, got[0].ravel()) assert_almost_equal(expected_decif, got[1].ravel(), decimal=4)
def fcts_model(X, y, max_depth, n_estimators, n_jobs): "RandomForestClassifier." rf = RandomForestClassifier(max_depth=max_depth, n_estimators=n_estimators, n_jobs=n_jobs) rf.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = convert_sklearn(rf, initial_types=initial_types, options={RandomForestClassifier: { 'zipmap': False }}) f = BytesIO() f.write(onx.SerializeToString()) content = f.getvalue() sess = InferenceSession(content) outputs = [o.name for o in sess.get_outputs()] def predict_skl_predict(X, model=rf): return rf.predict(X) def predict_skl_predict_proba(X, model=rf): return rf.predict_proba(X) def predict_onnxrt_predict(X, sess=sess): return sess.run(outputs[:1], {'X': X})[0] def predict_onnxrt_predict_proba(X, sess=sess): return sess.run(outputs[1:], {'X': X})[0] return { 'predict': (predict_skl_predict, predict_onnxrt_predict), 'predict_proba': (predict_skl_predict_proba, predict_onnxrt_predict_proba) }
def fcts_model(X, y, n_jobs): "LinearRegression." model = LinearRegression(n_jobs=n_jobs) model.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = convert_sklearn(model, initial_types=initial_types) sess = InferenceSession(onx.SerializeToString()) outputs = [o.name for o in sess.get_outputs()] oinf = OnnxInference(onx, runtime="python") def predict_skl_predict(X, model=model): return model.predict(X) def predict_onnxrt_predict(X, sess=sess): return sess.run(outputs[:1], {'X': X})[0] def predict_onnx_inference(X, oinf=oinf): return oinf.run({'X': X})["variable"] return { 'predict': (predict_skl_predict, predict_onnxrt_predict, predict_onnx_inference) }
clrrf = RandomForestClassifier(n_estimators=2, max_depth=2) clrrf.fit(X_train, y_train) clrrf.predict(X_test[:2]) paths, n_nodes_ptr = clrrf.decision_path(X_test[:2]) print(paths.todense()) model_def = to_onnx(clrrf, X_train.astype(numpy.float32), options={id(clrrf): {'decision_path': True, 'zipmap': False}}) sess = InferenceSession(model_def.SerializeToString()) ########################################## # The model produces 3 outputs. print([o.name for o in sess.get_outputs()]) ########################################## # Let's display the last one. res = sess.run(None, {'X': X_test[:2].astype(numpy.float32)}) print(res[-1]) ############################################################ # List of available options # +++++++++++++++++++++++++ # # Options are registered for every converted to detect any # supported options while running the conversion.
def predict(sess: rt.InferenceSession, data: list): input_name = sess.get_inputs()[0].name label_name = sess.get_outputs()[0].name return sess.run([label_name], { input_name: data })[0]
class OnnxTransformer(BaseEstimator, TransformerMixin, OnnxOperatorMixin): """ Calls :epkg:`onnxruntime` inference following :epkg:`scikit-learn` API so that it can be included in a :epkg:`scikit-learn` pipeline. Parameters ---------- onnx_bytes : bytes output_name: string requested output name or None to request all and have method *transform* to store all of them in a dataframe enforce_float32 : boolean :epkg:`onnxruntime` only supports *float32*, :epkg:`scikit-learn` usually uses double floats, this parameter ensures that every array of double floats is converted into single floats """ def __init__(self, onnx_bytes, output_name=None, enforce_float32=True): BaseEstimator.__init__(self) TransformerMixin.__init__(self) self.onnx_bytes = onnx_bytes self.output_name = output_name self.enforce_float32 = enforce_float32 if not isinstance(onnx_bytes, bytes): raise TypeError("onnx_bytes must be bytes to be pickled.") def __repr__(self): """ usual """ ob = self.onnx_bytes if len(ob) > 20: ob = ob[:10] + b"..." + ob[-10:] return "{0}(onnx_bytes=b'{1}', output_name={2}, enforce_float32={3})".format( self.__class__.__name__, ob, self.output_name, enforce_float32) def fit(self, X=None, y=None, **fit_params): """ Loads the :epkg:`ONNX` model. Parameters ---------- X : unused y : unused Returns ------- self """ self.onnxrt_ = InferenceSession(self.onnx_bytes) self.inputs_ = [_.name for _ in self.onnxrt_.get_inputs()] return self def _check_arrays(self, inputs): """ Ensures that double floats are converted into single floats if *enforce_float32* is True or raises an exception. """ for k in inputs: v = inputs[k] if isinstance(v, numpy.ndarray): if v.dtype == numpy.float64: if self.enforce_float32: inputs[k] = v.astype(numpy.float32) else: raise TypeError( "onnxunruntime only supports floats. Input '{0}' " "should be converted.".format(k)) def transform(self, X, y=None, **inputs): """ Runs the predictions. If *X* is a dataframe, the function assumes every columns is a separate input, otherwise, *X* is considered as a first input and *inputs* can be used to specify extra inputs. Parameters ---------- X : iterable, data to process (or first input if several expected) y : unused inputs: :epkg:`ONNX` graph support multiple inputs, each column of a dataframe is converted into as many inputs if *X* is a dataframe, otherwise, *X* is considered as the first input and *inputs* can be used to specify the other ones Returns ------- :epkg:`DataFrame` """ if not hasattr(self, "onnxrt_"): raise AttributeError( "Transform OnnxTransformer must be fit first.") rt_inputs = {} if isinstance(X, pandas.DataFrame): for c in X.columns: rt_inputs[c] = X[c] elif isinstance(X, numpy.ndarray): rt_inputs[self.inputs_[0]] = X elif isinstance(X, dict) and len(inputs) == 0: for k, v in X.items(): rt_inputs[k] = v elif isinstance(X, list): if len(self.inputs_) == 1: rt_inputs[self.inputs_[0]] = numpy.array(X) else: for i in range(len(self.inputs_)): rt_inputs[self.inputs_[i]] = [row[i] for row in X] for k, v in inputs.items(): rt_inputs[k] = v names = [self.output_name] if self.output_name else None self._check_arrays(rt_inputs) outputs = self.onnxrt_.run(names, rt_inputs) if self.output_name or len(outputs) == 1: if isinstance(outputs[0], list): return pandas.DataFrame(outputs[0]) else: return outputs[0] else: names = self.output_name if self.output_name else [ o.name for o in self.onnxrt_.get_outputs() ] return pandas.DataFrame({k: v for k, v in zip(names, outputs)}) def fit_transform(self, X, y=None, **inputs): """ Loads the *ONNX* model and runs the predictions. Parameters ---------- X : iterable, data to process (or first input if several expected) y : unused inputs: :epkg:`ONNX` graph support multiple inputs, each column of a dataframe is converted into as many inputs if *X* is a dataframe, otherwise, *X* is considered as the first input and *inputs* can be used to specify the other ones Returns ------- :epkg:`DataFrame` """ return self.fit(X, y=y, **inputs).transform(X, y) @staticmethod def enumerate_create(onnx_bytes, output_names=None, enforce_float32=True): """ Creates multiple *OnnxTransformer*, one for each requested intermediate node. onnx_bytes : bytes output_names: string requested output names or None to request all and have method *transform* to store all of them in a dataframe enforce_float32 : boolean :epkg:`onnxruntime` only supports *float32*, :epkg:`scikit-learn` usually uses double floats, this parameter ensures that every array of double floats is converted into single floats :return: iterator on OnnxTransformer *('output name', OnnxTransformer)* """ selected = None if output_names is None else set(output_names) model = load_onnx_model(onnx_bytes) for out in enumerate_model_node_outputs(model): m = select_model_inputs_outputs(model, out) if selected is None or out in selected: tr = OnnxTransformer(m.SerializeToString(), enforce_float32=enforce_float32) yield out, tr def onnx_parser(self, inputs=None): """ Returns a parser for this model. """ if inputs: self.parsed_inputs_ = inputs def parser(): return [o.name for o in self.onnxrt_.get_outputs()] return parser def onnx_shape_calculator(self): def shape_calculator(operator): cout = self.onnxrt_.get_outputs() if len(operator.outputs) != len(cout): raise RuntimeError("Mismatched number of outputs: {} != {}." "".format(len(operator.outputs), len(cout))) for out in operator.outputs: shape = out.type.shape typ = guess_type(out.type) out.type = typ(shape=shape) return shape_calculator def onnx_converter(self): """ Returns a converter for this model. If not overloaded, it fetches the converter mapped to the first *scikit-learn* parent it can find. """ inputs = getattr(self, "parsed_inputs_", None) if inputs is None: inputs = [] for inp in self.onnxrt_.get_inputs(): shape = inp.type.shape typ = guess_type(inp.type) inputs.append((inp.name, typ(shape))) if outputs is None: outputs = [out.name for out in self.onnxrt_.get_outputs()] def copy_inout(inout): shape = [s.dim_value for s in inout.type.tensor_type.shape.dim] value_info = helper.make_tensor_value_info( clean_name(inout.name), inout.type.tensor_type.elem_type, shape) return value_info def clean_variable_name(name, scope): return scope.get_unique_variable_name(naame) def clean_operator_name(name, scope): return scope.get_unique_operator_name(naame) def clean_initializer_name(name, scope): return scope.get_unique_variable_name(naame) def converter(scope, operator, container): graph = model_onnx.graph inputs = [copy_inout(o) for o in graph.input] outputs = [copy_inout(o) for o in graph.output] for node in graph.node: n = helper.make_node( node.op_type, [clean_variable_name(o) for o in node.input], [clean_variable_name(o) for o in node.output]) n.attribute.extend(node.attribute) # pylint: disable=E1101 container.nodes.append(n) inits = [] for o in graph.initializer: tensor = TensorProto() tensor.data_type = o.data_type tensor.name = clean_initializer_name(o.name) tensor.raw_data = o.raw_data tensor.dims.extend(o.dims) # pylint: disable=E1101 container.initializers.append(tensor) return converter
so.optimized_model_filepath = os.path.split(filename)[-1] + ".optimized.onnx" sess = InferenceSession(onx.SerializeToString(), so, providers=[provider]) bind = SessionIOBinding(sess._sess) print("graph_optimization_level:", so.graph_optimization_level) ##################################### # Creates random data feed = random_feed(sess, batch) ##################################### # moving the data on CPU or GPU feed_ort_value = OrderedDict( (name, (C_OrtValue.ortvalue_from_numpy(v, ort_device), v.dtype)) for name, v in feed.items()) outputs = [o.name for o in sess.get_outputs()] ####################################### # A function which calls the API for any device. def run_with_iobinding(sess, bind, ort_device, feed_ort_value, outputs): for name, (value, dtype) in feed_ort_value.items(): bind.bind_input(name, ort_device, dtype, value.shape(), value.data_ptr()) for out in outputs: bind.bind_output(out, ort_device) sess._sess.run_with_iobinding(bind, None) ortvalues = bind.get_outputs() return [o.numpy() for o in ortvalues]
# Add training parameter # ++++++++++++++++++++++ # new_onx = add_output_initializer( onx, ['C', 'l1_ratio'], [numpy.array([model.C]), numpy.array([model.l1_ratio])]) ######################################## # Inference # +++++++++ sess = InferenceSession(new_onx.SerializeToString(), providers=['CPUExecutionProvider']) print("output names:", [o.name for o in sess.get_outputs()]) res = sess.run(None, {'X': X_test[:2]}) print("outputs") pprint.pprint(res) ####################################### # The major draw back of this solution is increase the prediction # time as onnxruntime copies the constants for every prediction. # It is possible either to store those constant in a separate ONNX graph # or to removes them. # # Select outputs # ++++++++++++++ # # Next function removes unneeded outputs from a model, # not only the constants. Next model only keeps the probabilities.
class InferenceSession: # pylint: disable=E0102 """ Wrappers around InferenceSession from :epkg:`onnxruntime`. :param onnx_bytes: onnx bytes :param session_options: session options :param log_severity_level: change the logging level :param device: device, a string `cpu`, `cuda`, `cuda:0`... """ def __init__(self, onnx_bytes, sess_options=None, log_severity_level=4, device=None): if InferenceSession is None: raise ImportError( # pragma: no cover "onnxruntime is not available.") self.log_severity_level = log_severity_level if device is None: self.device = get_ort_device('cpu') else: self.device = get_ort_device(device) self.providers = device_to_providers(self.device) set_default_logger_severity(3) if sess_options is None: self.so = SessionOptions() self.so.log_severity_level = log_severity_level self.sess = OrtInferenceSession(onnx_bytes, sess_options=self.so, providers=self.providers) else: self.so = sess_options self.sess = OrtInferenceSession(onnx_bytes, sess_options=sess_options, providers=self.providers) self.ro = RunOptions() self.ro.log_severity_level = log_severity_level self.ro.log_verbosity_level = log_severity_level self.output_names = [o.name for o in self.get_outputs()] def run(self, output_names, input_feed, run_options=None): """ Executes the ONNX graph. :param output_names: None for all, a name for a specific output :param input_feed: dictionary of inputs :param run_options: None or RunOptions :return: array """ if any(map(lambda v: isinstance(v, C_OrtValue), input_feed.values())): return self.sess._sess.run_with_ort_values(input_feed, self.output_names, run_options or self.ro) return self.sess.run(output_names, input_feed, run_options or self.ro) def get_inputs(self): "Returns input types." return self.sess.get_inputs() def get_outputs(self): "Returns output types." return self.sess.get_outputs() def end_profiling(self): "Ends profiling." return self.sess.end_profiling()
def _create_onnx_graphs(self): """ Creates forward and backward ONNX graph. The new class has the following attributes: * `__doc__`: doc string * `__module__`: module name (this file) * `_run_options`: see :epkg:`RunOptions` * `_sess`: :epkg:`InferenceSession` with the original graph * `_sess_eval`: :epkg:`InferenceSession` on the graph with weights as inputs * `_training_agent`: :epkg:`TrainingAgent` * `_cache`: :epkg:`OrtValueCache` * `_logger`: logger * `_input_names`: input names * `_debug`: use debug mode * `_grad_input_names`: gradient input names * `_output_names`: output names * `_weights_to_train`: names of the weights to train Training attributes * `_bw_fetches_names`: bw_fetches_names, * `_fw_outputs_device_info`: fw_outputs_device_info, * `_bw_outputs_device_info`: bw_outputs_device_info, * `_fw_no_grad_output_device_info`: fw_no_grad_output_device_info, * `_graph_info`: graph_info} Additional attributes added if *keep_model* is True: * `_trained_onnx`: ONNX graph for the gradient * `_optimized_pre_grad_model`: evaluation ONNX graph taking weights as inputs * `_graph_builder`: :epkg:`OrtModuleGraphBuilder` """ logger = self._logger if logger is not None: logger.info("[OrtGradientForwardBackward] create training onnx") logger.info("[OrtGradientForwardBackward] input_names=%r", self.input_names) logger.info("[OrtGradientForwardBackward] output_names=%r", self.output_names) logger.info("[OrtGradientForwardBackward] weights_to_train=%r", self.weights_to_train) builder = OrtModuleGraphBuilder() if logger is not None: cf = self.graph_builder_config.graph_transformer_config cfp = cf.propagate_cast_ops_config logger.info("[OrtGradientForwardBackward] " "OrtModuleGraphBuilder.initialize") logger.info( "[OrtGradientForwardBackward] graph_builder_config=%s", OrtGradientForwardBackward._repr_helper_( self.graph_builder_config, indent=4)) logger.info( "[OrtGradientForwardBackward] graph_builder_config." "graph_transformer_config=%s", OrtGradientForwardBackward._repr_helper_(cf, indent=4)) logger.info( "[OrtGradientForwardBackward] graph_builder_config." "graph_transformer_config.propagate_cast_ops_config=%s", OrtGradientForwardBackward._repr_helper_(cfp, indent=4)) builder.initialize(self.onnx_model.SerializeToString(), self.graph_builder_config) if logger is not None: logger.info( "[OrtGradientForwardBackward] OrtModuleGraphBuilder.build") builder.build() if logger is not None: logger.info( "[OrtGradientForwardBackward] OrtModuleGraphBuilder.get_model") train_onnx_model_serialized = builder.get_model() optimized_pre_grad_model = builder.get_inference_optimized_model() graph_info = builder.get_graph_info() if logger is not None: logger.info( "[OrtGradientForwardBackward] graph_info=%s", OrtGradientForwardBackward._repr_helper_(graph_info, indent=4)) logger.info("[OrtGradientForwardBackward] create TrainSession") logger.info( "[OrtGradientForwardBackward] sess_options=%s", OrtGradientForwardBackward._repr_helper_(self.sess_options, indent=4)) logger.info("[OrtGradientForwardBackward] providers=%r", self.providers) sess = InferenceSession(train_onnx_model_serialized, sess_options=self.sess_options, provider_options=self.provider_options, providers=self.providers) if logger is not None: logger.info("[OrtGradientForwardBackward] create InferenceSession") sess_eval = InferenceSession(optimized_pre_grad_model, sess_options=self.sess_options, provider_options=self.provider_options, providers=self.providers) if logger is not None: logger.info("[OrtGradientForwardBackward] create training agent") grad_input_names = [obj.name for obj in sess.get_inputs()] bw_fetches_names = [obj.name for obj in sess.get_outputs()] fw_outputs_device_info = [ OrtDevice( OrtGradientForwardBackward._provider_name_to_device_type(i), OrtDevice.default_memory(), self.device_index) for i in self.providers ] bw_outputs_device_info = [ OrtDevice( OrtGradientForwardBackward._provider_name_to_device_type( self.providers[0]), OrtDevice.default_memory(), self.device_index) for i in bw_fetches_names ] fw_no_grad_output_device_info = [ OrtDevice( OrtGradientForwardBackward._provider_name_to_device_type( self.providers[0]), OrtDevice.default_memory(), self.device_index) for i in self.output_names ] try: # onnxruntime>=1.12 training_agent = TrainingAgent(sess._sess, grad_input_names, fw_outputs_device_info, bw_fetches_names, bw_outputs_device_info, 0) except TypeError: # onnxruntime<=1.11 training_agent = TrainingAgent(sess._sess, grad_input_names, fw_outputs_device_info, bw_fetches_names, bw_outputs_device_info) if logger is not None: logger.info( "[OrtGradientForwardBackward] instantiate dynamic class %r", self.class_name) logger.info("[OrtGradientForwardBackward] weights_to_train=%r", self.weights_to_train) logger.info("[OrtGradientForwardBackward] grad_input_names=%r", grad_input_names) logger.info("[OrtGradientForwardBackward] bw_fetches_names=%r", bw_fetches_names) logger.info("[OrtGradientForwardBackward] device_index=%r", self.device_index) devices = list(fw_outputs_device_info) while len(devices) < len(grad_input_names): devices.append(devices[-1]) trained_onnx = onnx.load(BytesIO(train_onnx_model_serialized)) onnx_loss = onnx.load(BytesIO(optimized_pre_grad_model)) for i, node in enumerate(trained_onnx.graph.node): if node.name == '': node.name = "N%d" % i for i, node in enumerate(onnx_loss.graph.node): if node.name == '': node.name = "N%d" % i kwargs = { '_run_options': self.run_options, '_sess': sess, '_sess_eval': sess_eval, '_training_agent': training_agent, '_cache': OrtValueCache(), '_logger': logger, '_input_names': self.input_names, '_grad_input_names': grad_input_names, '_output_names': self.output_names, '_bw_fetches_names': bw_fetches_names, '_fw_outputs_device_info': fw_outputs_device_info, '_bw_outputs_device_info': bw_outputs_device_info, '_fw_no_grad_output_device_info': fw_no_grad_output_device_info, '_weights_to_train': list(sorted(self.weights_to_train)), '_graph_info': graph_info, # '_trained_onnx': trained_onnx, '_optimized_pre_grad_model': onnx_loss, '_graph_builder': builder, '_devices': devices, '_debug': self.debug } graph = kwargs['_trained_onnx'].graph kwargs.update({ '_onx_inp': [o.name for o in graph.input], '_onx_out': [o.name for o in graph.output] }) if len(kwargs['_onx_inp']) != len(kwargs['_onx_out']): raise RuntimeError( # pragma: no cover "Gradient input and output are inconsistant: " "%r != %r" % (kwargs['_onx_inp'], kwargs['_onx_out'])) return kwargs