def test_add_loss_output_cls(self): from onnxcustom.utils.orttraining_helper import add_loss_output X, y = make_classification( # pylint: disable=W0632 100, n_features=10) X = X.astype(numpy.float32) y = y.astype(numpy.int64) X_train, X_test, y_train, y_test = train_test_split(X, y) reg = LogisticRegression() reg.fit(X_train, y_train) reg.coef_ = reg.coef_.reshape((1, -1)) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearClassifier'}, options={'zipmap': False}) onx_loss = add_loss_output(onx, 'log', output_index='probabilities', eps=1 - 6) try: text = onnx_simple_text_plot(onx_loss) except RuntimeError: text = "" if text: self.assertIn("Clip(probabilities", text) oinf = OnnxInference(onx_loss) output = oinf.run({'X': X_test, 'label': y_test.reshape((-1, 1))}) loss = output['loss'] skl_loss = log_loss(y_test, reg.predict_proba(X_test), eps=1 - 6) self.assertLess(numpy.abs(skl_loss - loss[0, 0]), 1e-5)
def print_graph(d): for k, v in sorted(d.items()): if isinstance(v, dict): print_graph(v) else: print("\n++++++", v.replace("\\", "/"), "\n") with open(v, "rb") as f: print(onnx_simple_text_plot(onnx.load(f)))
def plot_onnx(filename, format="onnx", verbose=0, output=None, fLOG=print): """ Plots an ONNX graph on the standard output. :param filename: onnx file :param format: format to export too (`simple`, `tree`, `dot`, `io`, `mat`, `raw`) :param output: output file to produce or None to print it on stdout :param verbose: verbosity level :param fLOG: logging function .. cmdref:: :title: Plots an ONNX graph as text :cmd: -m mlprodict plot_onnx --help :lid: l-cmd-plot_onnx The command shows the ONNX graphs as a text on the standard output. Example:: python -m mlprodict plot_onnx --filename="something.onnx" --format=simple """ if isinstance(filename, str): from onnx import load content = load(filename) else: content = filename if format == 'dot': from ..onnxrt import OnnxInference code = OnnxInference(filename).to_dot() elif format == 'simple': from mlprodict.plotting.text_plot import onnx_simple_text_plot code = onnx_simple_text_plot(content) elif format == 'io': from mlprodict.plotting.text_plot import onnx_text_plot_io code = onnx_text_plot_io(content) elif format == 'mat': from mlprodict.plotting.text_plot import onnx_text_plot code = onnx_text_plot(content) elif format == 'raw': code = str(content) elif format == 'tree': from mlprodict.plotting.plotting import onnx_text_plot_tree rows = [] for node in content.graph.node: if node.op_type.startswith("TreeEnsemble"): rows.append('Node type=%r name=%r' % (node.op_type, node.name)) rows.append(onnx_text_plot_tree(node)) code = "\n".join(rows) else: raise ValueError( # pragma: no cover "Unknown format %r." % format) if output not in ('', None): with open(output, "w", encoding="utf-8") as f: f.write(code) else: fLOG(code)
def wtest_ort_gradient_optimizers_fw_sgd_binary(self, use_weight): from onnxcustom.training.optimizers_partial import ( OrtGradientForwardBackwardOptimizer) from onnxcustom.training.sgd_learning_rate import (LearningRateSGD) from onnxcustom.training.sgd_learning_loss import NegLogLearningLoss X = numpy.arange(60).astype(numpy.float32).reshape((-1, 3)) y = numpy.arange(X.shape[0]).astype(numpy.float32).reshape( (-1, 1)) > 10 X = X.astype(numpy.float32) y = y.astype(numpy.int64) y[0, 0] = 0 y[-1, 0] = 1 w = (numpy.random.rand(y.shape[0]) + 1).astype(numpy.float32) X_train, _, y_train, __, w_train, ___ = train_test_split(X, y, w) reg = SGDClassifier(loss='log') if use_weight: reg.fit(X_train, y_train.ravel(), sample_weight=w_train.astype(numpy.float64)) else: reg.fit(X_train, y_train.ravel()) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}, options={ 'zipmap': False, 'raw_scores': True }) onx = select_model_inputs_outputs(onx, outputs=['score']) self.assertIn("output: name='score'", onnx_simple_text_plot(onx)) inits = ['coef', 'intercept'] train_session = OrtGradientForwardBackwardOptimizer( onx, inits, weight_name='weight' if use_weight else None, learning_rate=LearningRateSGD(1e10), learning_loss=NegLogLearningLoss(), warm_start=False, max_iter=100, batch_size=10, enable_logging=False) self.assertIsInstance(train_session.learning_loss, NegLogLearningLoss) self.assertEqual(train_session.learning_loss.eps, 1e-5) y_train = y_train.reshape((-1, 1)) if use_weight: train_session.fit(X_train, y_train, w_train.reshape((-1, 1))) else: train_session.fit(X_train, y_train) losses = train_session.train_losses_ self.assertGreater(len(losses), 1) if any(map(numpy.isnan, losses)): raise AssertionError(losses)
def wtest_ort_gradient_optimizers_fw_nesterov_binary_mlp( self, use_weight=True): from onnxcustom.training.optimizers_partial import ( OrtGradientForwardBackwardOptimizer) from onnxcustom.training.sgd_learning_rate import ( LearningRateSGDNesterov) from onnxcustom.training.sgd_learning_loss import NegLogLearningLoss X, y = make_classification( # pylint: disable=W0632 100, n_features=10, random_state=0) X = X.astype(numpy.float32) y = y.astype(numpy.int64) w = (numpy.random.rand(y.shape[0]) + 1).astype(numpy.float32) X_train, _, y_train, __, w_train, ___ = train_test_split(X, y, w) reg = MLPClassifier(solver='sgd') reg.fit(X_train, y_train) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}, options={'zipmap': False}) onx = select_model_inputs_outputs(onx, outputs=['out_activations_result']) self.assertIn("output: name='out_activations_result'", onnx_simple_text_plot(onx)) set_model_props(onx, {'info': 'unit test'}) onx = onnx_rename_weights(onx) inits = [ 'I0_coefficient', 'I1_intercepts', 'I2_coefficient1', 'I3_intercepts1' ] train_session = OrtGradientForwardBackwardOptimizer( onx, inits, weight_name='weight' if use_weight else None, learning_rate=LearningRateSGDNesterov(1e-4, nesterov=False, momentum=0.9), learning_loss=NegLogLearningLoss(), warm_start=False, max_iter=100, batch_size=10) self.assertIsInstance(train_session.learning_loss, NegLogLearningLoss) self.assertEqual(train_session.learning_loss.eps, 1e-5) if use_weight: train_session.fit(X_train, y_train, w_train) else: train_session.fit(X_train, y_train) temp = get_temp_folder( __file__, "temp_ort_gradient_optimizers_fw_nesterov_binary_mlp%d" % use_weight) train_session.save_onnx_graph(temp)
def test_grad_helper_loss(self): temp = get_temp_folder(__file__, "temp_grad_helper_loss") grad_file = os.path.join(temp, "grad.onnx") X, y = make_regression( # pylint: disable=W0632 100, n_features=10, bias=2, random_state=0) X = X.astype(numpy.float32) y = y.astype(numpy.float32) reg = LinearRegression() reg.fit(X, y) reg.coef_ = reg.coef_.reshape((1, -1)) onx = to_onnx(reg, X, target_opset=opset, black_op={'LinearRegressor'}) onx_loss = add_loss_output(onx) text1 = onnx_simple_text_plot(onx_loss) new_onx = onnx_derivative(onx, options=DerivativeOptions.Loss, label='variable', loss='loss', path_name=grad_file) text2 = onnx_simple_text_plot(new_onx) self.assertNotEqual(text1, text2)
def check_infer_shapes(self, onx, out, rt): onnx_shapes = infer_shapes(onx) inferred = onnx_shapes.graph.value_info # pylint: disable= for data in inferred: if data.name not in out: raise AssertionError("Name %r not found." % data.name) shape, dtype, sparse = OnnxShapeInference._get_shape(data) # pylint: disable=W0212 for i in range(len(shape)): if not isinstance(shape[i], str): continue if shape[i].startswith('unk_'): shape[i] = shape[i][4:] res = ShapeResult(data.name, shape, dtype, sparse) if res != out[data.name]: raise AssertionError( "Unexpected differences for name %r:\nexp: %r\ngot: %r" "\n-----\n%s" % (data.name, res, out[data.name], onnx_simple_text_plot(onx)))
def test_gradient_mlpclassifier(self): from onnxcustom.training.optimizers_partial import ( OrtGradientForwardBackwardOptimizer) from onnxcustom.training.sgd_learning_loss import NegLogLearningLoss X = numpy.arange(30).reshape((-1, 3)).astype(numpy.float32) / 100 y = numpy.arange(X.shape[0]).astype(numpy.float32) y = (y.reshape((-1, 1)) >= 15).astype(numpy.int64) reg = MLPClassifier(hidden_layer_sizes=(5,), max_iter=2, activation='logistic', momentum=0, nesterovs_momentum=False, alpha=0) reg.fit(X, y.ravel()) onx = to_onnx(reg, X, target_opset=opset, options={'zipmap': False}) onx = select_model_inputs_outputs(onx, outputs=['add_result1'], infer_shapes=True) text = onnx_simple_text_plot(onx) self.assertIn("output: name='add_result1'", text) onx = onnx_rename_weights(onx) inits = ["I0_coefficient", 'I1_intercepts', 'I2_coefficient1', 'I3_intercepts1'] xp = numpy.arange(2 * X.shape[1]).reshape((2, -1)).astype( numpy.float32) / 100 xp[0, 0] -= 4 xp[1, :] += 4 yp = numpy.array([0, 1], dtype=numpy.int64).reshape((-1, 1)) train_session = OrtGradientForwardBackwardOptimizer( onx, inits, learning_rate=1e-5, warm_start=True, max_iter=2, batch_size=10, learning_loss=NegLogLearningLoss()) train_session.fit(X, y) state = train_session.get_state() state_np = [st.numpy() for st in state] # gradient scikit-learn coef_grads = state_np[::2] intercept_grads = state_np[1::2] layer_units = [3, 5, 1] activations = [xp] + [None] * (len(layer_units) - 1) deltas = [None] * (len(activations) - 1) skl_pred = reg.predict_proba(xp) batch_loss, coef_grads, intercept_grads = reg._backprop( # pylint: disable=W0212 xp, yp, activations, deltas, coef_grads, intercept_grads) deltas = activations[-1] - yp # gradient onnxcustom ort_xp = C_OrtValue.ortvalue_from_numpy(xp, train_session.device) ort_yp = C_OrtValue.ortvalue_from_numpy(yp, train_session.device) ort_state = [ort_xp] + state prediction = train_session.train_function_.forward( ort_state, training=True) ort_pred = prediction[0].numpy() self.assertEqualArray(skl_pred[:, 1:2], expit(ort_pred), decimal=2) loss, loss_gradient = train_session.learning_loss.loss_gradient( train_session.device, ort_yp, prediction[0]) gradient = train_session.train_function_.backward([loss_gradient]) # comparison self.assertEqualArray( batch_loss * 2, loss.numpy(), decimal=3) self.assertEqualArray(deltas, loss_gradient.numpy(), decimal=3) # do not use iterator for gradient, it may crash ort_grad = [gradient[i].numpy() / xp.shape[0] for i in range(len(gradient))][1:] self.assertEqualArray( intercept_grads[1], ort_grad[3].ravel(), decimal=2) self.assertEqualArray(coef_grads[1], ort_grad[2], decimal=2) self.assertEqualArray( intercept_grads[0], ort_grad[1].ravel(), decimal=2) self.assertEqualArray(coef_grads[0], ort_grad[0], decimal=2)
def build_model(n_nodes, size, opv=15): OnnxAdd, OnnxIdentity = loadop('Add', 'Identity') x = 'X' for n in range(n_nodes): y = OnnxAdd(x, numpy.random.randn(size).astype(numpy.float32), op_version=opv) x = y final = OnnxIdentity(x, op_version=opv, output_names=['Y']) x = numpy.zeros((10, 10), dtype=numpy.float32) return final.to_onnx({'X': x}, {'Y': x}, target_opset=opv) model = build_model(2, 5) print(onnx_simple_text_plot(model)) ########################################## # Measure the time of serialization functions # +++++++++++++++++++++++++++++++++++++++++++ def parse(buffer): proto = onnx.ModelProto() proto.ParseFromString(buffer) return proto data = [] nodes = [5, 10, 20] for size in tqdm([10, 100, 1000, 10000, 100000, 200000, 300000]):
""" import numpy from scipy.special import expit import pandas from tqdm import tqdm from cpyquickhelper.numbers.speed_measure import measure_time import matplotlib.pyplot as plt from onnxruntime import InferenceSession from onnxruntime.capi._pybind_state import ( # pylint: disable=E0611 SessionIOBinding, OrtDevice as C_OrtDevice, OrtValue as C_OrtValue) from mlprodict.plotting.text_plot import onnx_simple_text_plot from onnxcustom.utils.onnx_function import function_onnx_graph fct_onx = function_onnx_graph("axpy") print(onnx_simple_text_plot(fct_onx)) ########################################### # The numpy implementation is the following. fct_numpy = lambda X1, X2, alpha: X1 * alpha + X2 ########################################### # The benchmark def reshape(a, dim): if len(a.shape) == 2: return a[:dim].copy() return a
def wtest_ort_gradient_optimizers_fw_nesterov_binary(self, use_weight): from onnxcustom.training.optimizers_partial import ( OrtGradientForwardBackwardOptimizer) from onnxcustom.training.sgd_learning_rate import ( LearningRateSGDNesterov) from onnxcustom.training.sgd_learning_loss import NegLogLearningLoss X, y = make_classification( # pylint: disable=W0632 100, n_features=10, random_state=0) X = X.astype(numpy.float32) y = y.astype(numpy.int64) w = (numpy.random.rand(y.shape[0]) + 1).astype(numpy.float32) X_train, _, y_train, __, w_train, ___ = train_test_split(X, y, w) reg = SGDClassifier(loss='log') if use_weight: reg.fit(X_train, y_train, sample_weight=w_train.astype(numpy.float64)) else: reg.fit(X_train, y_train) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}, options={ 'zipmap': False, 'raw_scores': True }) onx2 = onx onx = select_model_inputs_outputs(onx, outputs=['score']) self.assertIn("output: name='score'", onnx_simple_text_plot(onx)) set_model_props(onx, {'info': 'unit test'}) inits = ['coef', 'intercept'] train_session = OrtGradientForwardBackwardOptimizer( onx, inits, weight_name='weight' if use_weight else None, learning_rate=LearningRateSGDNesterov(1e-4, nesterov=False, momentum=0.9), learning_loss=NegLogLearningLoss(), warm_start=False, max_iter=100, batch_size=10) self.assertIsInstance(train_session.learning_loss, NegLogLearningLoss) self.assertEqual(train_session.learning_loss.eps, 1e-5) y_train = y_train.reshape((-1, 1)) if use_weight: train_session.fit(X_train, y_train, w_train.reshape((-1, 1))) else: train_session.fit(X_train, y_train) temp = get_temp_folder( __file__, "temp_ort_gradient_optimizers_fw_nesterov_binary") train_session.save_onnx_graph(temp) # get_trained_weight trained_onnx = train_session.get_trained_onnx(model=onx2) sess = InferenceSession(onx2.SerializeToString(), providers=['CPUExecutionProvider']) got1 = sess.run(None, {'X': X_train}) sess = InferenceSession(trained_onnx.SerializeToString(), providers=['CPUExecutionProvider']) got2 = sess.run(None, {'X': X_train}) self.assertEqual(len(got1), len(got2)) self.assertEqual(got1[0].shape, got2[0].shape) # state state = train_session.get_state() self.assertIsInstance(state, list) train_session.set_state(state) for k in range(len(state)): # pylint: disable=C0200 state[k] = state[k].numpy() train_session.set_state(state)