def fit(self, X=None, y=None, **fit_params): """ Loads the :epkg:`ONNX` model. :param X: unused :param y: unused :param fit_params: additional parameter (unused) :return: self """ from ..onnxrt.optim.onnx_helper import change_input_first_dimension onx = onnx.load(BytesIO(self.onnx_bytes)) output_names = set( o.name for o in onx.graph.output) # pylint: disable=E1101 updated = False if (self.output_name is not None and self.output_name not in output_names): # The model refers to intermediate outputs. onx = select_model_inputs_outputs( onx, outputs=[self.output_name]) updated = True if self.change_batch_size is not None: onx = change_input_first_dimension( onx, self.change_batch_size) updated = True onnx_bytes = ( onx.SerializeToString() if updated else self.onnx_bytes) self.onnxrt_ = OnnxInference(onnx_bytes, runtime=self.runtime) self.inputs_ = self.onnxrt_.input_names return self
def test_model_knn_regressor_radius(self): model, X = self._fit_model(RadiusNeighborsRegressor()) model_onnx = convert_sklearn(model, "KNN regressor", [("input", FloatTensorType([None, 4]))], target_opset=TARGET_OPSET, options={id(model): {'optim': 'cdist'}}) sess = InferenceSession(model_onnx.SerializeToString()) got = sess.run(None, {'input': X.astype(numpy.float32)})[0] exp = model.predict(X.astype(numpy.float32)) if any(numpy.isnan(got.ravel())): # The model is unexpectedly producing nan values # not on all platforms. rows = ['--EXP--', str(exp), '--GOT--', str(got), '--EVERY-OUTPUT--'] for out in enumerate_model_node_outputs( model_onnx, add_node=False): onx = select_model_inputs_outputs(model_onnx, out) sess = InferenceSession(onx.SerializeToString()) res = sess.run( None, {'input': X.astype(numpy.float32)}) rows.append('--{}--'.format(out)) rows.append(str(res)) if (StrictVersion(onnxruntime.__version__) < StrictVersion("1.4.0")): return raise AssertionError('\n'.join(rows)) assert_almost_equal(exp.ravel(), got.ravel(), decimal=3)
def _display_intermediate_steps(model_onnx, inputs): import onnxruntime print("[_display_intermediate_steps] BEGIN") if isinstance(model_onnx, str): import onnx model_onnx = onnx.load(model_onnx) for name, node in enumerate_model_initializers(model_onnx, add_node=True): print("INIT: {} - {}".format(name, _guess_type(node))) for out, node in enumerate_model_node_outputs(model_onnx, add_node=True): print('-') print("OUTPUT: {} from {}".format(out, node.name)) step = select_model_inputs_outputs(model_onnx, out) try: step_sess = onnxruntime.InferenceSession(step.SerializeToString()) except Exception as e: raise RuntimeError("Unable to load ONNX model with onnxruntime. " "Last added node is:\n{}".format(node)) from e for o in step_sess.get_inputs(): print("IN :", o) for o in step_sess.get_outputs(): print("OUT: ", o) if inputs: res = step_sess.run(inputs) print(res) print("[_display_intermediate_steps] END")
def _display_intermediate_steps(model_onnx, inputs, disable_optimisation): import onnxruntime print("[_display_intermediate_steps] BEGIN") if isinstance(model_onnx, str): import onnx model_onnx = onnx.load(model_onnx) for name, node in enumerate_model_initializers(model_onnx, add_node=True): print("INIT: {} - {}".format(name, _guess_type(node))) for out, node in enumerate_model_node_outputs(model_onnx, add_node=True): print('-') print("OUTPUT: {} from {}".format(out, node.name)) step = select_model_inputs_outputs(model_onnx, out) if (disable_optimisation and hasattr(onnxruntime, 'GraphOptimizationLevel')): opts = onnxruntime.SessionOptions() opts.graph_optimization_level = ( onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL) else: opts = None try: step_sess = onnxruntime.InferenceSession(step.SerializeToString(), sess_options=opts) except Exception as e: raise RuntimeError("Unable to load ONNX model with onnxruntime. " "Last added node is:\n{}".format(node)) from e for o in step_sess.get_inputs(): print("IN :", o) for o in step_sess.get_outputs(): print("OUT: ", o) if inputs: res = step_sess.run(inputs) print(res) print("[_display_intermediate_steps] END")
def save_model(model, feature_names, model_path, label_text="label"): p, extension = os.path.splitext(model_path) model.feature_names = feature_names pickle_path = p + ".pkl" if extension == ".pmml": try: from sklearn2pmml import sklearn2pmml, PMMLPipeline except ImportError: raise ImportError( "You need to install `sklearn2pmml` to store models in pmml format" ) pipeline = PMMLPipeline([("model", model)]) pipeline.target_field = label_text pipeline.active_fields = np.array(feature_names) sklearn2pmml(pipeline, model_path) elif extension == ".onnx": try: from skl2onnx import convert_sklearn from skl2onnx.common.data_types import FloatTensorType from skl2onnx.helpers.onnx_helper import select_model_inputs_outputs from onnx.onnx_pb import StringStringEntryProto except ImportError: raise ImportError( "You need to install `skl2onnx` to store models in onnx format" ) onnx = convert_sklearn( model, name=label_text, initial_types=[("input", FloatTensorType((None, len(feature_names))))], doc_string="Model created by aict-tools to estimate {}".format(label_text), ) # this makes sure we only get the scores and that they are numpy arrays and not # a list of dicts. # must come before setting metadata as it clears the metadata_props if hasattr(model, "predict_proba"): onnx = select_model_inputs_outputs(onnx, ["probabilities"]) metadata = dict( model_author="aict-tools", aict_tools_version=__version__, feature_names=",".join(feature_names), model_type="classifier" if is_classifier(model) else "regressor", ) for key, value in metadata.items(): onnx.metadata_props.append(StringStringEntryProto(key=key, value=value)) with open(model_path, "wb") as f: f.write(onnx.SerializeToString()) else: pickle_path = model_path # Always store the pickle dump,just in case joblib.dump(model, pickle_path, compress=4)
def print_specific_output(model_path, input_tensor, output_name, print_tensor=False): model_onnx = load_onnx_model(model_path) num_onnx = select_model_inputs_outputs(model_onnx, output_name) save_onnx_model(num_onnx, "remove_temp.onnx") sess = rt.InferenceSession("remove_temp.onnx") out_tensor = sess.run(None, input_tensor) print("name", output_name, "shape", out_tensor[0].shape) if print_tensor: print(out_tensor[0])
def test_onnx_helper_load_save_init_meta(self): model = make_pipeline(Binarizer(), OneHotEncoder(sparse=False), StandardScaler()) X = numpy.array([[0.1, 1.1], [0.2, 2.2], [0.4, 2.2], [0.2, 2.4]]) model.fit(X) model_onnx = convert_sklearn(model, "pipe3", [("input", FloatTensorType([None, 2]))]) meta = {'pA': 'one', 'pB': 'two'} onnx.helper.set_model_props(model_onnx, meta) new_model = select_model_inputs_outputs(model_onnx, "variable") vals = {p.key: p.value for p in new_model.metadata_props} assert vals == meta
def test_model_knn_regressor2_1_radius(self): model, X = self._fit_model_simple( RadiusNeighborsRegressor(algorithm="brute"), n_targets=2) X = X[:-1] model_onnx = convert_sklearn( model, "KNN regressor", [("input", FloatTensorType([None, X.shape[1]]))], target_opset=TARGET_OPSET) self.assertIsNotNone(model_onnx) sess = InferenceSession(model_onnx.SerializeToString()) got = sess.run(None, {'input': X.astype(numpy.float32)})[0] exp = model.predict(X.astype(numpy.float32)) if any(numpy.isnan(got.ravel())): # The model is unexpectedly producing nan values # not on all platforms. # It happens when two matrices are multiplied, # one is (2, 20, 20), second is (20, 20) # and contains only 0 or 1 values. # The output contains nan values on the first row # but not on the second one. rows = [ '--EXP--', str(exp), '--GOT--', str(got), '--EVERY-OUTPUT--' ] for out in enumerate_model_node_outputs(model_onnx, add_node=False): onx = select_model_inputs_outputs(model_onnx, out) sess = InferenceSession(onx.SerializeToString()) res = sess.run(None, {'input': X.astype(numpy.float32)}) rows.append('--{}--'.format(out)) rows.append(str(res)) if (onnxruntime.__version__.startswith('1.4.') or onnxruntime.__version__.startswith('1.5.')): # TODO: investigate the regression in onnxruntime 1.4 # One broadcasted multiplication unexpectedly produces nan. whole = '\n'.join(rows) if "[ nan" in whole: warnings.warn(whole) return raise AssertionError(whole) if (onnxruntime.__version__.startswith('1.3.') and sys.platform == 'win32'): # Same error but different line number for further # investigation. raise AssertionError(whole) raise AssertionError('\n'.join(rows)) assert_almost_equal(exp, got, decimal=5)
def _modify_model_add_outputs_nodes(self, model_dir): old_onnx_model = onnx.load(self.args.model_path) utils.print_info_log("load model success") for index, node in enumerate(old_onnx_model.graph.node): if not node.name: node.name = node.op_type + "_" + str(index) outputs_name = [ name for name in enumerate_model_node_outputs(old_onnx_model) ] new_onnx_model = select_model_inputs_outputs(old_onnx_model, outputs_name) new_onnx_model_path = os.path.join( model_dir, "new_" + os.path.basename(self.args.model_path)) save_onnx_model(new_onnx_model, new_onnx_model_path) utils.print_info_log("modify model outputs success") return old_onnx_model, new_onnx_model_path
def test_onnx_helper_load_save(self): model = make_pipeline(StandardScaler(), Binarizer(threshold=0.5)) X = numpy.array([[0.1, 1.1], [0.2, 2.2]]) model.fit(X) model_onnx = convert_sklearn(model, "binarizer", [("input", FloatTensorType([1, 2]))]) filename = "temp_onnx_helper_load_save.onnx" save_onnx_model(model_onnx, filename) model = load_onnx_model(filename) new_model = select_model_inputs_outputs(model, "variable") assert new_model.graph is not None tr1 = self.get_model(model) tr2 = self.get_model(new_model) X = X.astype(numpy.float32) X1 = tr1(X) X2 = tr2(X) assert X1.shape == (2, 2) assert X2.shape == (2, 2)
def test_onnx_helper_load_save_init(self): model = make_pipeline(Binarizer(), OneHotEncoder(sparse=False), StandardScaler()) X = numpy.array([[0.1, 1.1], [0.2, 2.2], [0.4, 2.2], [0.2, 2.4]]) model.fit(X) model_onnx = convert_sklearn(model, "pipe3", [("input", FloatTensorType([None, 2]))]) filename = "temp_onnx_helper_load_save.onnx" save_onnx_model(model_onnx, filename) model = load_onnx_model(filename) new_model = select_model_inputs_outputs(model, "variable") assert new_model.graph is not None tr1 = self.get_model(model) tr2 = self.get_model(new_model) X = X.astype(numpy.float32) X1 = tr1(X) X2 = tr2(X) assert X1.shape == (4, 2) assert X2.shape == (4, 2)
def test_onnx_helper_load_save(self): model = make_pipeline(StandardScaler(), Binarizer(threshold=0.5)) X = numpy.array([[0.1, 1.1], [0.2, 2.2]]) model.fit(X) model_onnx = convert_sklearn(model, 'binarizer', [('input', FloatTensorType([1, 2]))]) filename = "temp_onnx_helper_load_save.onnx" save_onnx_model(model_onnx, filename) model = load_onnx_model(filename) list(enumerate_model_node_outputs(model)) new_model = select_model_inputs_outputs(model, 'variable') self.assertTrue(new_model.graph is not None) tr1 = self.get_model(model) tr2 = self.get_model(new_model) X = X.astype(numpy.float32) X1 = tr1(X) X2 = tr2(X) self.assertEqual(X1.shape, (2, 2)) self.assertEqual(X2.shape, (2, 2))
def test_onnx_helper_load_save_init(self): model = make_pipeline(Binarizer(), OneHotEncoder(sparse=False), StandardScaler()) X = numpy.array([[0.1, 1.1], [0.2, 2.2], [0.4, 2.2], [0.2, 2.4]]) model.fit(X) model_onnx = convert_sklearn(model, 'pipe3', [('input', FloatTensorType([1, 2]))]) filename = "temp_onnx_helper_load_save.onnx" save_onnx_model(model_onnx, filename) model = load_onnx_model(filename) list(enumerate_model_node_outputs(model)) new_model = select_model_inputs_outputs(model, 'variable') self.assertTrue(new_model.graph is not None) # pylint: disable=E1101 tr1 = self.get_model(model) tr2 = self.get_model(new_model) X = X.astype(numpy.float32) X1 = tr1(X) X2 = tr2(X) self.assertEqual(X1.shape, (4, 2)) self.assertEqual(X2.shape, (4, 2))
def test_model_knn_regressor2_1_radius(self): model, X = self._fit_model_simple( RadiusNeighborsRegressor(algorithm="brute"), n_targets=2) model_onnx = convert_sklearn( model, "KNN regressor", [("input", FloatTensorType([None, X.shape[1]]))], target_opset=TARGET_OPSET) self.assertIsNotNone(model_onnx) sess = InferenceSession(model_onnx.SerializeToString()) got = sess.run(None, {'input': X.astype(numpy.float32)})[0] exp = model.predict(X.astype(numpy.float32)) if any(numpy.isnan(got.ravel())): # The model is unexpectedly producing nan values # not on all platforms. # It happens when two matrices are multiplied, # one is (2, 20, 20), second is (20, 20) # and contains only 0 or 1 values. # The output contains nan values on the first row # but not on the second one. rows = [ '--EXP--', str(exp), '--GOT--', str(got), '--EVERY-OUTPUT--' ] for out in enumerate_model_node_outputs(model_onnx, add_node=False): onx = select_model_inputs_outputs(model_onnx, out) sess = InferenceSession(onx.SerializeToString()) res = sess.run(None, {'input': X.astype(numpy.float32)}) rows.append('--{}--'.format(out)) rows.append(str(res)) if (StrictVersion(onnxruntime.__version__) < StrictVersion("1.4.0")): return raise AssertionError('\n'.join(rows)) assert_almost_equal(exp, got, decimal=5)
def enumerate_create(onnx_bytes, output_names=None, enforce_float32=True): """ Creates multiple *OnnxTransformer*, one for each requested intermediate node. onnx_bytes : bytes output_names: string requested output names or None to request all and have method *transform* to store all of them in a dataframe enforce_float32 : boolean :epkg:`onnxruntime` only supports *float32*, :epkg:`scikit-learn` usually uses double floats, this parameter ensures that every array of double floats is converted into single floats :return: iterator on OnnxTransformer *('output name', OnnxTransformer)* """ selected = None if output_names is None else set(output_names) model = load_onnx_model(onnx_bytes) for out in enumerate_model_node_outputs(model): m = select_model_inputs_outputs(model, out) if selected is None or out in selected: tr = OnnxTransformer(m.SerializeToString(), enforce_float32=enforce_float32) yield out, tr
# We need to modifies the *ONNX* before it is given to *onnxruntime*. # Let's see first the list of intermediate output. model_onnx = load_onnx_model("pipeline_titanic.onnx") for out in enumerate_model_node_outputs(model_onnx): print(out) ################################ # Not that easy to tell which one is what as the *ONNX* # has more operators than the original *scikit-learn* pipelines. # The graph at :ref:`l-plot-complex-pipeline-graph` # helps up to find the outputs of both numerical # and textual pipeline: *variable1*, *variable2*. # Let's look into the numerical pipeline first. num_onnx = select_model_inputs_outputs(model_onnx, 'variable1') save_onnx_model(num_onnx, "pipeline_titanic_numerical.onnx") ################################ # Let's compute the numerical features. sess = rt.InferenceSession("pipeline_titanic_numerical.onnx") numX = sess.run(None, inputs) print("numerical features", numX[0][:1]) ########################################### # We do the same for the textual features. print(model_onnx) text_onnx = select_model_inputs_outputs(model_onnx, 'variable2') save_onnx_model(text_onnx, "pipeline_titanic_textual.onnx")
inputs.append(onnx.load_tensor(input_file)) """ # For some models this string has to match the input to the model inputDict = {"image": inputs[0]} outputs_list = [] for out in enumerate_model_node_outputs(model_onnx): outputs_list.append(out) print(outputs_list) for idx, out in enumerate(outputs_list): name = str(idx) + "_" + out dataset = "test_data_set_0" os.mkdir(name) os.mkdir(name + "/" + dataset) modelPath = name + "/" + name + ".onnx" model_output = select_model_inputs_outputs(model_onnx, out) save_onnx_model(model_output, modelPath) sess = rt.InferenceSession(modelPath) numX = sess.run(None, inputDict) print() print("Generating idx=", idx) print(out) print(numX) # hardcoded for 1 output numpy_to_pb(out, numX[0], name + "/" + dataset + "/" + "output_0.pb") numpy_to_pb(out, inputs[0], name + "/" + dataset + "/" + "input_0.pb")
def benchmark_tflite(url, dest, onnx_name, opset, imgs, verbose=True, threshold=1e-3, names=None): """ Runs a simple benchmark with a tflite model. Goes through every steps (download, convert). Skips them if already done. """ if url.startswith('http'): tname = download_tflite(url, dest) if verbose: print("Created %r." % tname) else: tname = url # Converts the model. if verbose: print("Convert model in %r." % dest) convert_tflite(tname, onnx_name, opset) if verbose: print("Created %r." % onnx_name) # Benchmarks both models. ort = onnxruntime.InferenceSession(onnx_name) if verbose: print("ONNX inputs:") for a in ort.get_inputs(): print(" {}: {}, {}".format(a.name, a.type, a.shape)) print("ONNX outputs:") for a in ort.get_outputs(): print(" {}: {}, {}".format(a.name, a.type, a.shape)) # onnxruntime input_name = ort.get_inputs()[0].name fct_ort = lambda img: ort.run(None, {input_name: img})[0] results_ort, duration_ort = measure_time(fct_ort, imgs) if verbose: print("ORT", len(imgs), duration_ort) # tflite import tensorflow as tf interpreter = tf.lite.Interpreter(tname) #help(interpreter) input_details = interpreter.get_input_details() index_in = input_details[0]['index'] output_details = interpreter.get_output_details() index_out = output_details[0]['index'] interpreter.allocate_tensors() def call_tflite(inp): interpreter.set_tensor(index_in, inp) interpreter.invoke() scores = interpreter.get_tensor(index_out) return scores # check intermediate results if names is not None: from skl2onnx.helpers.onnx_helper import select_model_inputs_outputs import onnx with open(onnx_name, "rb") as f: model_onnx = onnx.load(f) interpreter_details = tf.lite.Interpreter( tname, experimental_preserve_all_tensors=True) input_details = interpreter_details.get_input_details() index_in = input_details[0]['index'] interpreter_details.allocate_tensors() interpreter_details.set_tensor(index_in, imgs[0]) interpreter_details.invoke() details = interpreter_details.get_tensor_details() inputs = {input_name: imgs[0]} names_index = {} for tt in details: names_index[tt['name']] = (tt['index'], tt['quantization'], tt['quantization_parameters']) num_results = [] for name_tfl, name_ort in names: index = names_index[name_tfl] tfl_value = interpreter_details.get_tensor(index[0]) new_name = onnx_name + ".%s.onnx" % name_ort.replace( ":", "_").replace(";", "_").replace("/", "_") if not os.path.exists(new_name): print('[create onnx model for %r, %r.' % (name_tfl, name_ort)) new_model = select_model_inputs_outputs(model_onnx, outputs=[name_ort]) with open(new_name, "wb") as f: f.write(new_model.SerializeToString()) ort_inter = onnxruntime.InferenceSession(new_name) result = ort_inter.run(None, inputs)[0] diff = numpy.abs(tfl_value.ravel().astype(numpy.float64) - result.ravel().astype(numpy.float64)).max() num_results.append("diff=%f names=(%r,%r) " % (diff, name_tfl, name_ort)) print("*** diff=%f names=(%r,%r) " % (diff, name_tfl, name_ort)) print(" TFL:", tfl_value.dtype, tfl_value.shape, tfl_value.min(), tfl_value.max()) print(" ORT:", result.dtype, result.shape, result.min(), result.max()) print("\n".join(num_results)) results_tfl, duration_tfl = measure_time(call_tflite, imgs) if verbose: print("TFL", len(imgs), duration_tfl) mean_ort = sum(duration_ort) / len(duration_ort) mean_tfl = sum(duration_tfl) / len(duration_tfl) print("ratio ORT=%r / TF=%r = %r" % (mean_ort, mean_tfl, mean_ort / mean_tfl)) # checks discrepencies res = call_tflite(imgs[0]) res_ort = fct_ort(imgs[0]) if isinstance(res, dict): if len(res) != 1: raise NotImplementedError( "TF output contains more than one output: %r." % res) output_name = ort.get_outputs()[0].name if output_name not in res: raise AssertionError("Unable to find output %r in %r." % (output_name, list(sorted(res)))) res = res[output_name] check_discrepencies(res_ort, res, threshold) return duration_ort, duration_tf
print("outputs") pprint.pprint(res) ####################################### # The major draw back of this solution is increase the prediction # time as onnxruntime copies the constants for every prediction. # It is possible either to store those constant in a separate ONNX graph # or to removes them. # # Select outputs # ++++++++++++++ # # Next function removes unneeded outputs from a model, # not only the constants. Next model only keeps the probabilities. simple_onx = select_model_inputs_outputs(new_onx, ['probabilities']) sess = InferenceSession(simple_onx.SerializeToString(), providers=['CPUExecutionProvider']) print("output names:", [o.name for o in sess.get_outputs()]) res = sess.run(None, {'X': X_test[:2]}) print("outputs") pprint.pprint(res) # Function *select_model_inputs_outputs* add also promote an intermediate # result to an output. # ##################################### # This example only uses ONNX graph in memory and never saves or loads a # model. This can be done by using the following snippets of code. #