def test_penalty_update(self): x = numpy.random.randn(10, 1).astype(numpy.float32) def fct(x): return numpy.sign(x) * 0.1 + (x * 0.9 * 2) exp_loss = x - fct(x) onx = function_onnx_graph('update_penalty_elastic_error', target_opset=get_max_opset(), dtype=numpy.float32, l1=0.1, l2=0.9) oinf = OnnxInference(onx) got = oinf.run({'X': x}) self.assertEqualArray(exp_loss, got['Y'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) got = sess.run(None, {'X': x}) self.assertEqualArray(exp_loss, got[0], decimal=5)
def test_penalty_3w(self): loss = numpy.random.randn(1, 1).astype(numpy.float32) w1 = numpy.random.randn(10, 1).astype(numpy.float32) w2 = numpy.random.randn(5, 1).astype(numpy.float32) def fct(x): return numpy.abs(x).sum() * 0.1 + ((x)**2).sum() * 0.9 exp_loss = loss + fct(w1) + fct(w2) onx = function_onnx_graph('n_penalty_elastic_error', target_opset=get_max_opset(), dtype=numpy.float32, n_tensors=2, l1_weight=0.1, l2_weight=0.9, weight_name='weight') oinf = OnnxInference(onx) got = oinf.run({'loss': loss, 'W0': w1, 'W1': w2}) self.assertEqualArray(exp_loss.reshape((-1, )), got['Y'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) got = sess.run(None, {'loss': loss, 'W0': w1, 'W1': w2}) self.assertEqualArray(exp_loss.reshape((-1, )), got[0], decimal=5)
def common_check(self, name, fct, weight_name=None, output_name='Y'): onx = function_onnx_graph(name, target_opset=get_max_opset(), dtype=numpy.float32, weight_name=weight_name) expected = numpy.random.randn(10, 1).astype(numpy.float32) predicted = numpy.random.randn(10, 1).astype(numpy.float32) w = numpy.random.rand(10).astype(numpy.float32) if weight_name is None: fin = fct(expected, predicted) else: fin = fct(expected, predicted, w) oinf = OnnxInference(onx) if weight_name is None: got = oinf.run({'X1': expected, 'X2': predicted}) else: got = oinf.run({'X1': expected, 'X2': predicted, 'weight': w}) self.assertEqualArray(fin, got[output_name], decimal=5) if weight_name is not None: got = oinf.run({'X1': expected, 'X2': predicted}) fin1 = fct(expected, predicted, numpy.array([1], dtype=expected.dtype)) self.assertEqualArray(fin1, got[output_name], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) if weight_name is None: got = sess.run(None, {'X1': expected, 'X2': predicted}) else: got = sess.run(None, { 'X1': expected, 'X2': predicted, 'weight': w }) self.assertEqualArray(fin, got[0], decimal=5) if weight_name is not None: got = sess.run(None, {'X1': expected, 'X2': predicted}) fin1 = fct(expected, predicted, numpy.array([1], dtype=expected.dtype)) self.assertEqualArray(fin1, got[0], decimal=5)
def common_unary(self, name, fct): onx = function_onnx_graph(name, target_opset=get_max_opset(), dtype=numpy.float32) x = numpy.random.randn(10, 1).astype(numpy.float32) fin = fct(x) oinf = OnnxInference(onx) got = oinf.run({'X': x}) self.assertEqualArray(fin, got['Y'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) got = sess.run(None, {'X': x}) self.assertEqualArray(fin, got[0], decimal=5)
def common_check_1(self, name, fct, weight_name=None, **kwargs): onx = function_onnx_graph(name, target_opset=get_max_opset(), dtype=numpy.float32, weight_name=weight_name, **kwargs) x = numpy.random.randn(10, 1).astype(numpy.float32) exp_loss, exp_grad = fct(x) oinf = OnnxInference(onx) got = oinf.run({'X': x}) self.assertEqualArray(exp_loss, got['Y'], decimal=5) self.assertEqualArray(exp_grad, got['Y_grad'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) got = sess.run(None, {'X': x}) self.assertEqualArray(exp_loss, got[0], decimal=5) self.assertEqualArray(exp_grad, got[1], decimal=5)
def common_check_alpha_beta(self, name, fct): onx = function_onnx_graph(name, target_opset=get_max_opset(), dtype=numpy.float32) x1 = numpy.random.randn(10, 1).astype(numpy.float32) x2 = numpy.random.randn(10, 1).astype(numpy.float32) g = numpy.random.randn(10, 1).astype(numpy.float32) alpha = numpy.random.randn(1).astype(numpy.float32) beta = numpy.random.randn(1).astype(numpy.float32) y, z = fct(x1, x2, g, alpha, beta) oinf = OnnxInference(onx) got = oinf.run({ 'X1': x1, 'X2': x2, 'alpha': alpha, 'beta': beta, 'G': g }) self.assertEqualArray(y, got['Y'], decimal=5) self.assertEqualArray(z, got['Z'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) got = sess.run(None, { 'X1': x1, 'X2': x2, 'alpha': alpha, 'beta': beta, 'G': g }) self.assertEqualArray(y, got[0], decimal=5) self.assertEqualArray(z, got[1], decimal=5)
def common_check_3(self, name, fct): onx = function_onnx_graph(name, target_opset=get_max_opset(), dtype=numpy.float32) x = numpy.random.randn(10, 1).astype(numpy.float32) a = numpy.random.randn(10, 1).astype(numpy.float32).T b = numpy.random.randn(10, 1).astype(numpy.float32) y = fct(x, a, b) code = export2onnx(onx) self.assertIn("'OnnxAdd'", code) oinf = OnnxInference(onx) got = oinf.run({'X': x, 'A': a, 'B': b}) self.assertEqualArray(y, got['Y'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) got = sess.run(None, {'X': x, 'A': a, 'B': b}) self.assertEqualArray(y, got[0], decimal=5)
def common_check_2(self, name, fct, weight_name=None, verbose=0, classification=False, rnd=True, second_name='Y_grad', **kwargs): onx = function_onnx_graph(name, target_opset=get_max_opset(), dtype=numpy.float32, weight_name=weight_name, **kwargs) if verbose > 0: with open(name + ".onnx", "wb") as f: f.write(onx.SerializeToString()) if classification: N = 10 p = numpy.random.randn(N, 1).astype(numpy.float32) p[0, :] = 0 p[1, :] = 100 p[2, :] = -100 p[3, :] = 1 p[4, :] = -1 y = (numpy.random.randn(N, 1).astype(numpy.float32) > 0).astype( numpy.int64) x2 = p x1 = y else: if rnd: x1 = numpy.random.randn(10, 1).astype(numpy.float32) x2 = numpy.random.randn(10, 1).astype(numpy.float32) else: x1 = numpy.zeros((10, 1), dtype=numpy.float32) x2 = numpy.zeros((10, 1), dtype=numpy.float32) + 1 if rnd: w = numpy.random.rand(10).astype(numpy.float32) else: w = numpy.zeros(10, dtype=numpy.float32) + 0.2 if weight_name is None: exp_loss, exp_grad = fct(x1, x2) else: exp_loss, exp_grad = fct(x1, x2, w.reshape((-1, 1))) oinf = OnnxInference(onx) run_params = dict(verbose=verbose, fLOG=print) if verbose > 0 else {} if verbose > 0: print(f"\n+++++ name(1)={name!r}") if weight_name is None: got = oinf.run({'X1': x1, 'X2': x2}, **run_params) else: got = oinf.run({'X1': x1, 'X2': x2, 'weight': w}, **run_params) self.assertEqual(len(exp_grad.shape), 2) self.assertEqual(exp_grad.shape[-1], 1) self.assertEqualArray(exp_grad, got[second_name], decimal=5) self.assertEqualArray(exp_loss, got['Y'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 0 if verbose > 0 else 4 so.log_verbosity_level = 0 if verbose > 0 else 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) if verbose > 0: print("+++ run") if weight_name is None: got = sess.run(None, {'X1': x1, 'X2': x2}) else: got = sess.run(None, {'X1': x1, 'X2': x2, 'weight': w}) self.assertEqualArray(exp_loss, got[0], decimal=5) self.assertEqualArray(exp_grad, got[1], decimal=5) if weight_name is not None: if verbose > 0: print("+++ run*") got = sess.run(None, {'X1': x1, 'X2': x2}) exp_loss2, exp_grad2 = fct(x1, x2, numpy.array([1], dtype=x1.dtype)) self.assertEqualArray(exp_loss2, got[0], decimal=5) self.assertEqualArray(exp_grad2, got[1], decimal=5) if 'grad' in name: rew = unreduced_onnx_loss(onx) if 'ReduceSum' in str(rew): raise AssertionError(f"Isse with:\n{rew!r}") if verbose > 0: with open(name + ".unreduced.onnx", "wb") as f: f.write(rew.SerializeToString()) if verbose > 0: print(f"\n+++++ name(2)={name!r}") oinf = OnnxInference(rew) if weight_name is None: got = oinf.run({'X1': x1, 'X2': x2}, **run_params) else: got = oinf.run({'X1': x1, 'X2': x2, 'weight': w}, **run_params) score = got['score'] self.assertEqual(len(score.shape), 2) self.assertEqual(score.shape[0], 10) self.assertEqual(score.shape[1], 1) self.assertEqualFloat(exp_loss, score.sum()) sess = InferenceSession(rew.SerializeToString(), so, providers=providers) if verbose > 0: print("+++ run") if weight_name is None: got = sess.run(None, {'X1': x1, 'X2': x2}) else: got = sess.run(None, {'X1': x1, 'X2': x2, 'weight': w}) score = got[0] self.assertEqual(len(score.shape), 2) self.assertEqual(score.shape[0], 10) self.assertEqual(score.shape[1], 1) self.assertEqualFloat(exp_loss, score.sum())
def forward_no_training(self, exc=None, verbose=False): if exc is None: exc = __name__ != '__main__' from onnxruntime.capi._pybind_state import (OrtValue as C_OrtValue, OrtDevice as C_OrtDevice, OrtMemType) from onnxruntime.capi._pybind_state import (OrtValueVector) from onnxcustom.training.ortgradient import OrtGradientForwardBackward X, y = make_regression( # pylint: disable=W0632 100, n_features=10, bias=2) X = X.astype(numpy.float32) y = y.astype(numpy.float32) X_train, X_test, y_train, _ = train_test_split(X, y) reg = LinearRegression() reg.fit(X_train, y_train) reg.coef_ = reg.coef_.reshape((1, -1)) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}) # starts testing if verbose: print("[forward_no_training] start testing") if exc: if verbose: print("[forward_no_training] check exception") self.assertRaise( lambda: OrtGradientForwardBackward( onx, debug=True, enable_logging=True, providers=['NONE']), ValueError) if verbose: print("[forward_no_training] instantiate") forback = OrtGradientForwardBackward(onx, debug=True, enable_logging=True) self.assertEqual(repr(forback), "OrtGradientForwardBackward(...)") self.assertTrue(hasattr(forback, 'cls_type_')) self.assertEqual(forback.cls_type_._onx_inp, ['X', 'coef', 'intercept']) self.assertEqual(forback.cls_type_._onx_out, ['X_grad', 'coef_grad', 'intercept_grad']) self.assertEqual(forback.cls_type_._weights_to_train, ['coef', 'intercept']) self.assertEqual(forback.cls_type_._grad_input_names, ['X', 'coef', 'intercept']) self.assertEqual(forback.cls_type_._input_names, ['X']) self.assertEqual(forback.cls_type_._bw_fetches_names, ['X_grad', 'coef_grad', 'intercept_grad']) self.assertEqual(forback.cls_type_._output_names, ['variable']) if verbose: print("[forward_no_training] expected prediction") expected = reg.predict(X_test) coef = reg.coef_.astype(numpy.float32).reshape((-1, 1)) intercept = numpy.array([reg.intercept_], dtype=numpy.float32) if verbose: print("[forward_no_training] InferenceSession") providers = device_to_providers('cpu') sess0 = InferenceSession(onx.SerializeToString(), providers=providers) inames = [i.name for i in sess0.get_inputs()] # pylint: disable=E1101 self.assertEqual(inames, ['X']) got = sess0.run(None, {'X': X_test}) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) if verbose: print("[forward_no_training] evaluation") sess_eval = forback.cls_type_._sess_eval # pylint: disable=E1101 inames = [i.name for i in sess_eval.get_inputs()] self.assertEqual(inames, ['X', 'coef', 'intercept']) got = sess_eval.run(None, { 'X': X_test, 'coef': coef, 'intercept': intercept }) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) # OrtValue if verbose: print("[forward_no_training] OrtValue") inst = forback.new_instance() device = C_OrtDevice(C_OrtDevice.cpu(), OrtMemType.DEFAULT, 0) # list of OrtValues inputs = [] for a in [X_test, coef, intercept]: inputs.append(C_OrtValue.ortvalue_from_numpy(a, device)) got_ort = inst.forward(inputs) got = [v.numpy() for v in got_ort] self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) # OrtValueVector if verbose: print("[forward_no_training] OrtValueVector") inputs = OrtValueVector() for a in [X_test, coef, intercept]: inputs.push_back(C_OrtValue.ortvalue_from_numpy(a, device)) got = inst.forward(inputs) self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].numpy().ravel(), decimal=4) # numpy if verbose: print("[forward_no_training] numpy") inputs = [X_test, coef, intercept] got = inst.forward(inputs) self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].numpy().ravel(), decimal=4) if verbose: print("[forward_no_training] end")
def forward_training(self, model, debug=False, n_classes=3, add_print=False): from onnxruntime.capi._pybind_state import (OrtValue as C_OrtValue, OrtMemType, OrtDevice as C_OrtDevice) from onnxruntime.capi._pybind_state import (OrtValueVector) from onnxcustom.training.ortgradient import OrtGradientForwardBackward def to_proba(yt): mx = yt.max() + 1 new_yt = numpy.zeros((yt.shape[0], mx), dtype=numpy.float32) for i, y in enumerate(yt): new_yt[i, y] = 1 return new_yt if hasattr(model.__class__, 'predict_proba'): X, y = make_classification( # pylint: disable=W0632 100, n_features=10, n_classes=n_classes, n_informative=7) X = X.astype(numpy.float32) y = y.astype(numpy.int64) else: X, y = make_regression( # pylint: disable=W0632 100, n_features=10, bias=2) X = X.astype(numpy.float32) y = y.astype(numpy.float32) X_train, X_test, y_train, y_test = train_test_split(X, y) reg = model reg.fit(X_train, y_train) # needs if skl2onnx<1.10.4 # reg.coef_ = reg.coef_.reshape((1, -1)) # reg.intercept_ = reg.intercept_.reshape((-1, )) if hasattr(model.__class__, 'predict_proba'): onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearClassifier'}, options={'zipmap': False}) onx = select_model_inputs_outputs( onx, outputs=[onx.graph.output[1].name]) else: onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}) # remove batch possibility #onx.graph.input[0].type.tensor_type.shape.dim[0].dim_value = 0 #onx.graph.input[0].type.tensor_type.shape.dim[0].dim_param = "batch_size" #onx.graph.output[0].type.tensor_type.shape.dim[0].dim_value = 0 #onx.graph.output[0].type.tensor_type.shape.dim[0].dim_param = "batch_size" providers = device_to_providers('cpu') sess = InferenceSession(onx.SerializeToString(), providers=providers) sess.run(None, {'X': X_test[:1]}) # starts testing forback = OrtGradientForwardBackward(onx, debug=True, enable_logging=True) if debug: n = model.__class__.__name__ temp = get_temp_folder(__file__, f"temp_forward_training_{n}") with open(os.path.join(temp, f"model_{n}.onnx"), "wb") as f: f.write(onx.SerializeToString()) with open(os.path.join(temp, f"fw_train_{n}.onnx"), "wb") as f: f.write(forback.cls_type_._trained_onnx.SerializeToString()) with open(os.path.join(temp, f"fw_pre_{n}.onnx"), "wb") as f: gr = forback.cls_type_._optimized_pre_grad_model f.write(gr.SerializeToString()) if hasattr(model.__class__, 'predict_proba'): expected = reg.predict_proba(X_test) coef = reg.coef_.astype(numpy.float32).T intercept = reg.intercept_.astype(numpy.float32) # only one observation X_test1 = X_test[:1] y_test = to_proba(y_test).astype(numpy.float32) y_test1 = y_test[:1] expected1 = expected[:1] else: expected = reg.predict(X_test) coef = reg.coef_.astype(numpy.float32).reshape((-1, 1)) intercept = numpy.array([reg.intercept_], dtype=numpy.float32) # only one observation X_test1 = X_test[:1] y_test1 = y_test[0].reshape((1, -1)) expected1 = expected[:1] # OrtValueVector inst = forback.new_instance() device = C_OrtDevice(C_OrtDevice.cpu(), OrtMemType.DEFAULT, 0) if add_print: print("\n\n######################\nFORWARD") inputs = OrtValueVector() for a in [X_test1, coef, intercept]: inputs.push_back(C_OrtValue.ortvalue_from_numpy(a, device)) got = inst.forward(inputs, training=True) self.assertEqual(len(got), 1) self.assertEqualArray(expected1.ravel(), got[0].numpy().ravel(), decimal=4) if add_print: print("\n\n######################\nBACKWARD") outputs = OrtValueVector() outputs.push_back(C_OrtValue.ortvalue_from_numpy(y_test1, device)) got = inst.backward(outputs) self.assertEqual(len(got), 3) if add_print: print("\n######################\nEND\n") # OrtValueVectorN inputs = OrtValueVector() for a in [X_test, coef, intercept]: inputs.push_back(C_OrtValue.ortvalue_from_numpy(a, device)) got = inst.forward(inputs, training=True) self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].numpy().ravel(), decimal=4) outputs = OrtValueVector() outputs.push_back( C_OrtValue.ortvalue_from_numpy(y_test.reshape((1, -1)), device)) got = inst.backward(outputs) self.assertEqual(len(got), 3) # list of OrtValues inputs = [] for a in [X_test, coef, intercept]: inputs.append(C_OrtValue.ortvalue_from_numpy(a, device)) got_ort = inst.forward(inputs, training=True) got = [v.numpy() for v in got_ort] self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) outputs = [ C_OrtValue.ortvalue_from_numpy(y_test.reshape((1, -1)), device) ] got = inst.backward(outputs) self.assertEqual(len(got), 3) # numpy inputs = [X_test, coef, intercept] got_ort = inst.forward(inputs, training=True) got = [v.numpy() for v in got_ort] self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) outputs = [y_test.reshape((1, -1))] got = inst.backward(outputs) self.assertEqual(len(got), 3)
def test_forward_no_training_pickle(self): from onnxruntime.capi._pybind_state import (OrtValue as C_OrtValue, OrtMemType, OrtDevice as C_OrtDevice) from onnxruntime.capi._pybind_state import (OrtValueVector) from onnxcustom.training.ortgradient import OrtGradientForwardBackward X, y = make_regression( # pylint: disable=W0632 100, n_features=10, bias=2) X = X.astype(numpy.float32) y = y.astype(numpy.float32) X_train, X_test, y_train, _ = train_test_split(X, y) reg = LinearRegression() reg.fit(X_train, y_train) reg.coef_ = reg.coef_.reshape((1, -1)) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}) forback0 = OrtGradientForwardBackward(onx, debug=True) st = io.BytesIO() pickle.dump(forback0, st) st2 = io.BytesIO(st.getvalue()) forback = pickle.load(st2) self.assertTrue(hasattr(forback, 'cls_type_')) self.assertEqual(forback.cls_type_._onx_inp, ['X', 'coef', 'intercept']) self.assertEqual(forback.cls_type_._onx_out, ['X_grad', 'coef_grad', 'intercept_grad']) self.assertEqual(forback.cls_type_._weights_to_train, ['coef', 'intercept']) self.assertEqual(forback.cls_type_._grad_input_names, ['X', 'coef', 'intercept']) self.assertEqual(forback.cls_type_._input_names, ['X']) self.assertEqual(forback.cls_type_._bw_fetches_names, ['X_grad', 'coef_grad', 'intercept_grad']) self.assertEqual(forback.cls_type_._output_names, ['variable']) expected = reg.predict(X_test) coef = reg.coef_.astype(numpy.float32).reshape((-1, 1)) intercept = numpy.array([reg.intercept_], dtype=numpy.float32) providers = device_to_providers('cpu') sess0 = InferenceSession(onx.SerializeToString(), providers=providers) inames = [i.name for i in sess0.get_inputs()] self.assertEqual(inames, ['X']) got = sess0.run(None, {'X': X_test}) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) sess_eval = forback.cls_type_._sess_eval # pylint: disable=W0212 inames = [i.name for i in sess_eval.get_inputs()] self.assertEqual(inames, ['X', 'coef', 'intercept']) got = sess_eval.run(None, { 'X': X_test, 'coef': coef, 'intercept': intercept }) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) # OrtValue inst = forback.new_instance() inputs = [] device = C_OrtDevice(C_OrtDevice.cpu(), OrtMemType.DEFAULT, 0) for a in [X_test, coef, intercept]: inputs.append(C_OrtValue.ortvalue_from_numpy(a, device)) got_ort = inst.forward(inputs) got = [v.numpy() for v in got_ort] self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].ravel(), decimal=4) # OrtValueVector inputs = OrtValueVector() for a in [X_test, coef, intercept]: inputs.push_back(C_OrtValue.ortvalue_from_numpy(a, device)) got = inst.forward(inputs) self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].numpy().ravel(), decimal=4) # numpy inputs = [X_test, coef, intercept] got = inst.forward(inputs) self.assertEqual(len(got), 1) self.assertEqualArray(expected.ravel(), got[0].numpy().ravel(), decimal=4)
def test_device_to_provider(self): self.assertEqual(device_to_providers('cpu'), ['CPUExecutionProvider']) self.assertEqual(device_to_providers('gpu'), ['CUDAExecutionProvider']) self.assertRaise(lambda: device_to_providers('NONE'), ValueError)