def test_onnxt_runtime_solve(self): for transposed in [False, True]: with self.subTest(transposed=transposed): A = numpy.array([[2, 1], [0, 1]], dtype=float) Y = numpy.array([2, 1], dtype=float) X = solve(A, Y, transposed=transposed) onx = OnnxSolve('A', 'Y', output_names=['X'], transposed=transposed, op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx( { 'A': A.astype(numpy.float32), 'Y': Y.astype(numpy.float32) }, outputs={'X': X.astype(numpy.float32)}, target_opset=get_opset_number_from_onnx()) oinf = OnnxInference(model_def) got = oinf.run({'A': A, 'Y': Y}) self.assertEqual(list(sorted(got)), ['X']) self.assertEqualArray(X, got['X'], decimal=6) python_tested.append(OnnxCDist) oinfpy = OnnxInference(model_def, runtime="python", inplace=True) validate_python_inference(oinfpy, { 'A': A.astype(numpy.float32), 'Y': Y.astype(numpy.float32) }) python_tested.append(OnnxSolve)
def test_onnxt_runtime_cdist(self): for metric in ['sqeuclidean', 'euclidean']: with self.subTest(metric=metric): X = numpy.array([[2, 1], [0, 1]], dtype=float) Y = numpy.array([[2, 1, 5], [0, 1, 3]], dtype=float).T Z = cdist(X, Y, metric=metric) onx = OnnxCDist('X', 'Y', output_names=['Z'], metric=metric, op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx( { 'X': X.astype(numpy.float32), 'Y': Y.astype(numpy.float32) }, outputs={'Z': Z.astype(numpy.float32)}, target_opset=get_opset_number_from_onnx()) self.assertIn('s: "%s"' % metric, str(model_def)) oinf = OnnxInference(model_def) got = oinf.run({'X': X, 'Y': Y}) self.assertEqual(list(sorted(got)), ['Z']) self.assertEqualArray(Z, got['Z'], decimal=6) python_tested.append(OnnxCDist) oinfpy = OnnxInference(model_def, runtime="python", inplace=True) validate_python_inference(oinfpy, { 'X': X.astype(numpy.float32), 'Y': Y.astype(numpy.float32) }, tolerance=1e-6)
def test_code_add_transpose(self): idi = numpy.identity(2, dtype=numpy.float32) onx = OnnxTranspose(OnnxAdd('X', idi, op_version=get_opset_number_from_onnx()), output_names=['Y'], op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx({'X': idi.astype(numpy.float32)}) oinf = OnnxInference(model_def, runtime='python') res = oinf.to_python(inline=False) self.assertNotEmpty(res) self.assertIsInstance(res, dict) self.assertEqual(len(res), 2) self.assertIn('onnx_pyrt_Ad_Addcst.pkl', res) self.assertIn('onnx_pyrt_main.py', res) cd = res['onnx_pyrt_main.py'] self.assertIn('def pyrt_Add(X, Ad_Addcst):', cd) self.assertIn('def run(self, X):', cd) # inline temp = get_temp_folder(__file__, "temp_code_add_transpose") res = oinf.to_python(inline=True, dest=temp) self.assertNotEmpty(res) name = os.path.join(temp, 'onnx_pyrt_main.py') self.assertExists(name) # test code test_code = """ X = numpy.array([[1, 2], [3, 4]], dtype=numpy.float32) oinf = OnnxPythonInference() Y = oinf.run(X) print(Y) """ X = numpy.array([[1, 2], [3, 4]], dtype=numpy.float32) exp = oinf.run({'X': X})['Y'] sexp = str(exp) self.auto_test_script(name, test_code, sexp)
def test_validate_sklearn_operators_all(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") logger = getLogger('skl2onnx') logger.disabled = True verbose = 1 if __name__ == "__main__" else 0 temp = get_temp_folder(__file__, "temp_validate_sklearn_operators_all") if False: # pylint: disable=W0125 rows = list( enumerate_validated_operator_opsets( verbose, models={"DecisionTreeClassifier"}, filter_exp=lambda m, p: '64' not in p, debug=True, fLOG=fLOG)) else: rows = list( enumerate_validated_operator_opsets( verbose, debug=None, fLOG=fLOG, dump_folder=temp, time_kwargs={ get_opset_number_from_onnx(): dict(number=2, repeat=2) }, n_features=[None])) self.assertGreater(len(rows), 1) df = DataFrame(rows) self.assertGreater(df.shape[1], 1) fLOG("output results") df.to_csv(os.path.join(temp, "sklearn_opsets_report.csv"), index=False) df.to_excel(os.path.join(temp, "sklearn_opsets_report.xlsx"), index=False)
def test_create_asv_benchmark_pyspy_compiled(self): self.assertNotEmpty(mlprodict) temp = get_temp_folder(__file__, "temp_create_asv_benchmark_pyspy_compiled") created = create_asv_benchmark(location=temp, verbose=0, runtime=('python', 'python_compiled'), exc=False, execute=True, models={'AdaBoostRegressor'}, add_pyspy=True) self.assertNotEmpty(created) ops = get_opset_number_from_onnx() verif = False allnames = [] for path, _, files in os.walk(os.path.join(temp, 'pyspy')): for zoo in files: if '__init__' in zoo: continue allnames.append(zoo) fullname = os.path.join(path, zoo) with open(fullname, 'r', encoding='utf-8') as f: content = f.read() if (zoo.endswith( "bench_AdaBoostReg_default_b_reg_nest10_1_4_%d_float_.py" % ops) and compare_module_version( sklearn.__version__, "0.21") >= 0): if "setup_profile" not in content: raise AssertionError(content) verif = True if not verif: raise AssertionError("Visited files\n{}".format( "\n".join(allnames)))
def test_template_benchmark_classifier_raw_scores(self): if not os.path.exists('_cache'): os.mkdir('_cache') cl = TemplateBenchmarkClassifierRawScore() res = {} cl.setup_cache() N = 60 nf = cl.params[2][1] opset = get_opset_number_from_onnx() dtype = 'float' optim = None for runtime in ['skl', 'pyrt', 'ort']: cl.setup(runtime, N, nf, opset, dtype, optim) self.assertEqual(cl.X.shape, (N, nf)) for method in dir(cl): if method.split('_')[0] in ('time', 'peakmem', 'track'): meth = getattr(cl.__class__, method) res[method, runtime] = meth(cl, runtime, N, nf, opset, dtype, optim) if method == 'track_score' and res[method, runtime] in (0, 1): raise AssertionError( "Predictions are too perfect: {},{}: {}".format( method, runtime, res[method, runtime])) self.assertEqual(len(res), 24) exp = [('time_predict', 'skl'), ('peakmem_predict', 'skl'), ('track_score', 'skl'), ('track_onnxsize', 'skl'), ('time_predict', 'pyrt'), ('peakmem_predict', 'pyrt'), ('track_score', 'pyrt'), ('track_onnxsize', 'pyrt'), ('time_predict', 'ort'), ('peakmem_predict', 'ort'), ('track_score', 'ort'), ('track_onnxsize', 'ort'), ('track_nbnodes', 'skl'), ('track_nbnodes', 'ort'), ('track_nbnodes', 'pyrt')] self.assertEqual(set(_ for _ in exp if not _[0].startswith('track_v')), set(_ for _ in res if not _[0].startswith('track_v')))
def test_onnxt_runtime_add_raise(self): idi = numpy.identity(2) onx = OnnxAdd('X', idi, output_names=['Y'], op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx({'X': idi.astype(numpy.float32)}) self.assertRaise(lambda: OnnxInference(model_def, runtime='onnxruntime-1'), ValueError)
def test_onnx_test_knn_transform(self): iris = load_iris() X, _ = iris.data, iris.target X_train, X_test = train_test_split(X, random_state=11) clr = NearestNeighbors(n_neighbors=3) clr.fit(X_train) for to in (10, 11, 12): if to > get_opset_number_from_onnx(): break try: model_def = to_onnx( clr, X_train.astype(numpy.float32), rewrite_ops=True, options={NearestNeighbors: { 'largest0': False }}, target_opset=to) except NameError as e: if "Option 'largest0' not in" in str(e): continue oinf = OnnxInference(model_def, runtime='python') X_test = X_test[:3] y = oinf.run({'X': X_test.astype(numpy.float32)}) dist, ind = clr.kneighbors(X_test) self.assertEqual(list(sorted(y)), ['distance', 'index']) self.assertEqualArray(ind, y['index']) self.assertEqualArray(dist, DataFrame(y['distance']).values, decimal=5)
def __init__(self, estimator, dim=None, N_fit=100000, runtimes=('python_compiled', 'onnxruntime1'), onnx_options=None, dtype=numpy.float32, **opts): """ @param estimator estimator class @param dim number of features @param N_fit number of observations to fit an estimator @param runtimes runtimes to test for class :epkg:`OnnxInference` @param opts training settings @param onnx_options ONNX conversion options @param dtype dtype (float32 or float64) """ # These libraries are optional. from skl2onnx import to_onnx # pylint: disable=E0401,C0415 from skl2onnx.common.data_types import FloatTensorType, DoubleTensorType # pylint: disable=E0401,C0415 if dim is None: raise RuntimeError( # pragma: no cover "dim must be defined.") BenchPerfTest.__init__(self, **opts) allowed = {"max_depth"} opts = {k: v for k, v in opts.items() if k in allowed} self.dtype = dtype self.skl = estimator(**opts) X, y = self._get_random_dataset(N_fit, dim) try: self.skl.fit(X, y) except Exception as e: # pragma: no cover raise RuntimeError( "X.shape={}\nopts={}\nTraining failed for {}".format( X.shape, opts, self.skl)) from e if dtype == numpy.float64: initial_types = [('X', DoubleTensorType([None, X.shape[1]]))] elif dtype == numpy.float32: initial_types = [('X', FloatTensorType([None, X.shape[1]]))] else: raise ValueError( # pragma: no cover "Unable to convert the model into ONNX, unsupported dtype {}.". format(dtype)) self.logconvert = StringIO() with contextlib.redirect_stdout(self.logconvert): with contextlib.redirect_stderr(self.logconvert): onx = to_onnx(self.skl, initial_types=initial_types, options=onnx_options, target_opset=get_opset_number_from_onnx()) onx.ir_version = get_ir_version_from_onnx() self._init(onx, runtimes)
def extract_model_info_onnx(self, **kwargs): """ Populates member ``self.onnx_info`` with additional information on the :epkg:`ONNX` graph. """ self.onnx_info = { 'onnx_nodes': len(self.ort_onnx.graph.node), # pylint: disable=E1101 'onnx_opset': get_opset_number_from_onnx(), } self.onnx_info.update(kwargs)
def test_onnx_example_cdist_in_euclidean(self): x = numpy.array([1, 2, 4, 5, 5, 4]).astype(numpy.float32).reshape( (3, 2)) x2 = numpy.array([1.1, 2.1, 4.01, 5.01, 5.001, 4.001, 0, 0]).astype(numpy.float32).reshape((4, 2)) cop = OnnxAdd('input', 'input', op_version=get_opset_number_from_onnx()) cop2 = OnnxIdentity(onnx_cdist( cop, x2, dtype=numpy.float32, metric='euclidean', op_version=get_opset_number_from_onnx()), output_names=['cdist'], op_version=get_opset_number_from_onnx()) model_def = cop2.to_onnx(inputs=[('input', FloatTensorType([None, None]))], outputs=[('cdist', FloatTensorType())], target_opset=get_opset_number_from_onnx()) sess = OnnxInference(model_def) res = sess.run({'input': x})['cdist'] exp = scipy_cdist(x * 2, x2, metric="euclidean") self.assertEqualArray(exp, res, decimal=5) x = numpy.array( [[6.1, 2.8, 4.7, 1.2], [5.7, 3.8, 1.7, 0.3], [7.7, 2.6, 6.9, 2.3], [6.0, 2.9, 4.5, 1.5], [6.8, 2.8, 4.8, 1.4], [5.4, 3.4, 1.5, 0.4], [5.6, 2.9, 3.6, 1.3], [6.9, 3.1, 5.1, 2.3]], dtype=numpy.float32) cop = OnnxAdd('input', 'input', op_version=get_opset_number_from_onnx()) cop2 = OnnxIdentity(onnx_cdist( cop, x, dtype=numpy.float32, op_version=get_opset_number_from_onnx()), output_names=['cdist'], op_version=get_opset_number_from_onnx()) model_def = cop2.to_onnx(inputs=[('input', FloatTensorType([None, None]))], outputs=[('cdist', FloatTensorType())], target_opset=get_opset_number_from_onnx()) sess = OnnxInference(model_def) res = sess.run({'input': x})['cdist'] exp = scipy_cdist(x * 2, x, metric="sqeuclidean") self.assertEqualArray(exp, res, decimal=4)
def test_onnxt_runtime_add1(self): idi = numpy.identity(2, dtype=numpy.float32) onx = OnnxAdd('X', idi, output_names=['Y'], op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx({'X': idi.astype(numpy.float32)}) X = numpy.array([[1, 2], [3, 4]], dtype=numpy.float32) model_def.ir_version = get_ir_version_from_onnx() oinf = OnnxInference(model_def, runtime='onnxruntime1') got = oinf.run({'X': X}) self.assertEqual(list(sorted(got)), ['Y']) self.assertEqualArray(idi + X, got['Y'], decimal=6)
def test_onnx_example_cdist_in_minkowski(self): x = numpy.array([1, 2, 1, 3, 2, 2, 2, 3]).astype(numpy.float32).reshape((4, 2)) x2 = numpy.array([[1, 2], [2, 2], [2.1, 2.1], [2, 2]]).astype(numpy.float32).reshape((4, 2)) for pp in [1, 2]: with self.subTest(pp=pp): cop = OnnxIdentity('input', op_version=get_opset_number_from_onnx()) cop2 = OnnxIdentity(onnx_cdist( cop, x2, dtype=numpy.float32, metric="minkowski", p=pp, op_version=get_opset_number_from_onnx()), output_names=['cdist'], op_version=get_opset_number_from_onnx()) model_def = cop2.to_onnx(inputs=[('input', FloatTensorType([None, None]))], outputs=[('cdist', FloatTensorType()) ]) try: sess = OnnxInference(model_def) except RuntimeError as e: raise AssertionError("Issue\n{}".format(model_def)) from e res = sess.run({'input': x})['cdist'] exp = scipy_cdist(x, x2, metric="minkowski", p=pp) self.assertEqualArray(exp, res, decimal=5) with self.subTest(pp=3): x = numpy.array([[6.1, 2.8, 4.7, 1.2], [5.7, 3.8, 1.7, 0.3], [7.7, 2.6, 6.9, 2.3], [6.0, 2.9, 4.5, 1.5], [6.8, 2.8, 4.8, 1.4], [5.4, 3.4, 1.5, 0.4], [5.6, 2.9, 3.6, 1.3], [6.9, 3.1, 5.1, 2.3]], dtype=numpy.float32) cop = OnnxAdd('input', 'input', op_version=get_opset_number_from_onnx()) cop2 = OnnxIdentity(onnx_cdist( cop, x, dtype=numpy.float32, metric="minkowski", p=3, op_version=get_opset_number_from_onnx()), output_names=['cdist'], op_version=get_opset_number_from_onnx()) model_def = cop2.to_onnx(inputs=[('input', FloatTensorType([None, None]))], outputs=[('cdist', FloatTensorType())]) sess = OnnxInference(model_def) res = sess.run({'input': x})['cdist'] exp = scipy_cdist(x * 2, x, metric="minkowski", p=3) self.assertEqualArray(exp, res, decimal=4)
def test_code_add_except(self): idi = numpy.identity(2, dtype=numpy.float32) onx = OnnxAdd('X', idi, output_names=['Y'], op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx({'X': idi.astype(numpy.float32)}) model_def.ir_version = get_ir_version_from_onnx() oinf = OnnxInference(model_def, runtime='onnxruntime1') try: oinf.to_python() except ValueError: pass
def test_cpu_conv_init(self): x = numpy.random.rand(1, 96, 56, 56).astype(numpy.float32) W = numpy.random.rand(24, 96, 1, 1).astype(numpy.float32) onx = OnnxConv('X', 'W', output_names=['Y'], auto_pad='NOTSET', group=1, dilations=[1, 1], kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1], op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx( { 'X': x.astype(numpy.float32), 'W': W.astype(numpy.float32) }, target_opset=get_opset_number_from_onnx()) oinf = OnnxInference(model_def) oinfrt = OnnxInference(model_def, runtime='onnxruntime1') for _ in range(0, 3): x = numpy.random.rand(1, 96, 56, 56).astype(numpy.float32) W = numpy.random.rand(24, 96, 1, 1).astype(numpy.float32) got = oinf.run({'X': x, 'W': W}) gotrt = oinfrt.run({'X': x, 'W': W}) diff = list(numpy.abs((gotrt['Y'] - got['Y']).ravel())) sdiff = list(sorted(diff)) if sdiff[-1] > 1e-5: raise AssertionError("runtimes disagree {}".format(sdiff[-5:])) for ii in range(len(diff)): # pylint: disable=C0200 if numpy.isnan(diff[ii]): raise AssertionError( "runtimes disagree about nan {}: {} # {} ? {}".format( ii, diff[ii], gotrt['Y'].ravel()[ii], got['Y'].ravel()[ii])) self.assertEqualArray(gotrt['Y'], got['Y'], decimal=5)
def test_onnxruntime_bug(self): rnd = numpy.random.randn(2, 20, 20).astype(numpy.float32) bni = (numpy.random.random((20, 20)).astype( # pylint: disable=E1101 numpy.float32) >= 0.7).astype(numpy.float32) mul = rnd * bni isn = any(numpy.isnan(mul.ravel())) self.assertFalse(isn) node = OnnxMul('X', bni, output_names=['Y'], op_version=get_opset_number_from_onnx()) onx = node.to_onnx({'X': rnd}) for rt in ['python', 'onnxruntime1']: with self.subTest(runtime=rt): oinf = OnnxInference(onx, runtime=rt) y = oinf.run({'X': rnd})['Y'] self.assertEqualArray(mul, y)
def test_model_bernoulli_nb_bc_onnxruntime1(self): model, X = self.fit_classification_model(BernoulliNB(), 2) model_onnx = convert_sklearn( model, "?", [("input", FloatTensorType([None, X.shape[1]]))], target_opset=get_opset_number_from_onnx()) exp1 = model.predict(X) exp = model.predict_proba(X) model_onnx.ir_version = get_ir_version_from_onnx() oinf = _capture_output( lambda: OnnxInference(model_onnx, runtime='onnxruntime1'), 'c')[0] got = oinf.run({'input': X}) self.assertEqualArray(exp1, got['output_label']) got2 = DataFrame(got['output_probability']).values self.assertEqualArray(exp, got2, decimal=4)
def test_model_extra_trees_classifier_multilabel(self): model, X_test = fit_multilabel_classification_model( ExtraTreesClassifier(random_state=42, n_estimators=10)) options = {id(model): {'zipmap': False}} model_onnx = convert_sklearn( model, "scikit-learn ExtraTreesClassifier", [("input", FloatTensorType([None, X_test.shape[1]]))], options=options, target_opset=get_opset_number_from_onnx()) self.assertTrue(model_onnx is not None) self.assertNotIn('zipmap', str(model_onnx).lower()) dump_data_and_model( X_test, model, model_onnx, basename="SklearnExtraTreesClassifierMultiLabel-Out0", folder=self.folder)
def test_validate_sklearn_operators_dump_all(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") logger = getLogger('skl2onnx') logger.disabled = True verbose = 1 if __name__ == "__main__" else 0 temp = get_temp_folder(__file__, "temp_validate_sklearn_operators_dump_all") self.assertRaise( lambda: list( enumerate_validated_operator_opsets( verbose, models={"DecisionTreeClassifier"}, filter_exp=lambda m, p: '64' not in p, fLOG=fLOG, dump_all=True)), ValueError) rows = list( enumerate_validated_operator_opsets( verbose, models={"DecisionTreeClassifier"}, filter_exp=lambda m, p: '64' not in p, fLOG=fLOG, dump_all=True, dump_folder=temp)) self.assertGreater(len(rows), 1) df = DataFrame(rows) self.assertGreater(df.shape[1], 1) fLOG("output results") df.to_csv(os.path.join(temp, "sklearn_opsets_report.csv"), index=False) df.to_excel(os.path.join(temp, "sklearn_opsets_report.xlsx"), index=False) stored = os.path.join( temp, ("dump-i-python-DecisionTreeClassifier-default-b-cl-tree._classes." "DecisionTreeClassifierzipmapFalse-op%d-nf4.pkl" % get_opset_number_from_onnx())) with open(stored, "rb") as f: obj = pickle.load(f) self.assertIn('onnx_bytes', obj) self.assertIn('skl_model', obj) self.assertIn('X_test', obj) self.assertIn('Xort_test', obj)
def test_validate_sklearn_operators_onnxruntime_KMeans(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") logger = getLogger('skl2onnx') logger.disabled = True verbose = 1 if __name__ == "__main__" else 0 buffer = [] def myprint(*args, **kwargs): buffer.append(" ".join(map(str, args))) op = get_opset_number_from_onnx() rows = list(enumerate_validated_operator_opsets( verbose, models={"KMeans"}, fLOG=myprint, runtime='onnxruntime2', debug=True, filter_exp=lambda m, p: '-64' not in p, opset_min=op, opset_max=op)) self.assertGreater(len(rows), 1)
def test_template_benchmark_transformPositive(self): if not os.path.exists('_cache'): os.mkdir('_cache') cl = TemplateBenchmarkTransformPositive() res = {} cl.setup_cache() N = 60 nf = cl.params[2][1] opset = get_opset_number_from_onnx() dtype = 'float' expect = 12 optim = None for runtime in ['skl', 'pyrt']: try: cl.setup(runtime, N, nf, opset, dtype, optim) except MissingShapeCalculator: # Converter not yet implemented. expect = 0 continue self.assertEqual(cl.X.shape, (N, nf)) for method in dir(cl): if method.split('_')[0] in ('time', 'peakmem', 'track'): meth = getattr(cl.__class__, method) res[method, runtime] = meth(cl, runtime, N, nf, opset, dtype, optim) if method == 'track_score' and res[method, runtime] in (0, 1): raise AssertionError( "Predictions are too perfect: {},{}: {}".format( method, runtime, res[method, runtime])) if expect == 0: return self.assertEqual(len(res), expect) exp = [('time_predict', 'skl'), ('peakmem_predict', 'skl'), ('track_score', 'skl'), ('track_onnxsize', 'skl'), ('time_predict', 'pyrt'), ('peakmem_predict', 'pyrt'), ('track_score', 'pyrt'), ('track_onnxsize', 'pyrt'), ('track_nbnodes', 'skl'), ('track_opset', 'skl'), ('track_opset', 'pyrt'), ('track_nbnodes', 'pyrt')] self.assertEqual(set(_ for _ in exp if not _[0].startswith('track_v')), set(_ for _ in res if not _[0].startswith('track_v')))
def test_onnxruntime_knn_radius(self): def _get_reg_data(self, n, n_features, n_targets, n_informative=10): X, y = make_regression( # pylint: disable=W0632 n, n_features=n_features, random_state=0, n_targets=n_targets, n_informative=n_informative) return X, y def _fit_model(model, n_targets=1, label_int=False, n_informative=10): X, y = _get_reg_data(20, 4, n_targets, n_informative) if label_int: y = y.astype(numpy.int64) model.fit(X, y) return model, X model, X = _fit_model(RadiusNeighborsRegressor()) model_onnx = to_onnx( model, X[:1].astype(numpy.float32), target_opset=get_opset_number_from_onnx(), options={id(model): {'optim': 'cdist'}}) oinf = OnnxInference(model_onnx, runtime='onnxruntime1') X = X[:7] got = oinf.run({'X': X.astype(numpy.float32)})['variable'] exp = model.predict(X.astype(numpy.float32)) if any(numpy.isnan(got.ravel())): # The model is unexpectedly producing nan values # sometimes. res = oinf.run({'X': X.astype(numpy.float32)}, intermediate=True) rows = ['--EXP--', str(exp), '--GOT--', str(got), '--EVERY-OUTPUT--'] for k, v in res.items(): rows.append('-%s-' % k) rows.append(str(v)) if any(map(numpy.isnan, res["variable"].ravel())): raise AssertionError('\n'.join(rows)) # onnxruntime and mlprodict do not return the same # output warnings.warn('\n'.join(rows)) return self.assertEqualArray(exp, got, decimal=4)
class TemplateBenchmarkClassifier(_CommonAsvSklBenchmarkClassifier): """ :epkg:`asv` test for a classifier, Full template can be found in `common_asv_skl.py <https://github.com/sdpython/mlprodict/ blob/master/mlprodict/asv_benchmark/common_asv_skl.py>`_. """ params = [ ['skl', 'pyrtc', 'ort'], # values for runtime [1, 10, 100, 1000, 10000, 100000], # values for N [4, 20], # values for nf [get_opset_number_from_onnx()], # values for opset ['float', 'double'], # values for dtype [None], # values for optim ] # additional parameters def setup_cache(self): # pylint: disable=W0235 super().setup_cache() def _create_model(self): return LogisticRegression(multi_class='ovr', solver='liblinear')
class TemplateBenchmarkTransform(_CommonAsvSklBenchmarkTransform): """ :epkg:`asv` example for a transform, Full template can be found in `common_asv_skl.py <https://github.com/sdpython/mlprodict/blob/ master/mlprodict/asv_benchmark/common_asv_skl.py>`_. """ params = [ ['skl', 'pyrtc', 'ort'], # values for runtime [1, 10, 100, 1000, 10000, 100000], # values for N [4, 20], # values for nf [get_opset_number_from_onnx()], # values for opset ['float', 'double'], # values for dtype [None], # values for optim ] # additional parameters def setup_cache(self): # pylint: disable=W0235 super().setup_cache() def _create_model(self): return Normalizer()
def test_validate_sklearn_operators_benchmark_errors(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") logger = getLogger('skl2onnx') logger.disabled = True verbose = 1 if __name__ == "__main__" else 0 temp = get_temp_folder( __file__, "temp_validate_sklearn_operators_benchmark_summary") rows = list( enumerate_validated_operator_opsets( verbose, models={"RFE", "DecisionTreeRegressor"}, opset_min=10, benchmark=True, fLOG=fLOG)) self.assertGreater(len(rows), 1) df = DataFrame(rows) for col in ['skl', 'batch']: self.assertIn('lambda-' + col, df.columns) for col in ['1', '10']: self.assertIn('time-ratio-N=' + col, df.columns) self.assertGreater(df.shape[1], 1) self.assertGreater(df.loc[0, "tostring_time"], 0) piv = summary_report(df) self.assertGreater(piv.shape[1], 1) self.assertIn('RT/SKL-N=1', piv.columns) self.assertNotIn('RT/SKL-N=10', piv.columns) self.assertIn('N=10', piv.columns) fLOG("output results") ops = 'opset%d' % get_opset_number_from_onnx() li = len(piv[ops].notnull()) self.assertEqual(li, piv.shape[0]) df.to_excel(os.path.join(temp, "sklearn_opsets_report.xlsx"), index=False) piv.to_excel(os.path.join(temp, "sklearn_opsets_summary.xlsx"), index=False)
def _enumerate_asv_benchmark_all_models( # pylint: disable=R0914 location, opset_min=10, opset_max=None, runtime=('scikit-learn', 'python'), models=None, skip_models=None, extended_list=True, n_features=None, dtype=None, verbose=0, filter_exp=None, dims=None, filter_scenario=None, exc=True, flat=False, execute=False, dest_pyspy=None, fLOG=print): """ Loops over all possible models and fills a folder with benchmarks following :epkg:`asv` concepts. :param n_features: number of features to try :param dims: number of observations to try :param verbose: integer from 0 (None) to 2 (full verbose) :param opset_min: tries every conversion from this minimum opset :param opset_max: tries every conversion up to maximum opset :param runtime: runtime to check, *scikit-learn*, *python*, *onnxruntime1* to check :epkg:`onnxruntime`, *onnxruntime2* to check every ONNX node independently with onnxruntime, many runtime can be checked at the same time if the value is a comma separated list :param models: list of models to test or empty string to test them all :param skip_models: models to skip :param extended_list: extends the list of :epkg:`scikit-learn` converters with converters implemented in this module :param n_features: change the default number of features for a specific problem, it can also be a comma separated list :param dtype: '32' or '64' or None for both, limits the test to one specific number types :param fLOG: logging function :param filter_exp: function which tells if the experiment must be run, None to run all, takes *model, problem* as an input :param filter_scenario: second function which tells if the experiment must be run, None to run all, takes *model, problem, scenario, extra* as an input :param exc: if False, raises warnings instead of exceptions whenever possible :param flat: one folder for all files or subfolders :param execute: execute each script to make sure imports are correct :param dest_pyspy: add a file to profile the prediction function with :epkg:`pyspy` """ ops = [_ for _ in sklearn_operators(extended=extended_list)] patterns = _read_patterns() if models is not None: if not all(map(lambda m: isinstance(m, str), models)): raise ValueError( "models must be a set of strings.") # pragma: no cover ops_ = [_ for _ in ops if _['name'] in models] if len(ops) == 0: raise ValueError( "Parameter models is wrong: {}\n{}".format( # pragma: no cover models, ops[0])) ops = ops_ if skip_models is not None: ops = [m for m in ops if m['name'] not in skip_models] if verbose > 0: def iterate(): for i, row in enumerate(ops): # pragma: no cover fLOG("{}/{} - {}".format(i + 1, len(ops), row)) yield row if verbose >= 11: verbose -= 10 # pragma: no cover loop = iterate() # pragma: no cover else: try: from tqdm import trange def iterate_tqdm(): with trange(len(ops)) as t: for i in t: row = ops[i] disp = row['name'] + " " * (28 - len(row['name'])) t.set_description("%s" % disp) yield row loop = iterate_tqdm() except ImportError: # pragma: no cover loop = iterate() else: loop = ops if opset_max is None: opset_max = get_opset_number_from_onnx() opsets = list(range(opset_min, opset_max + 1)) all_created = set() # loop on all models for row in loop: model = row['cl'] problems, extras = _retrieve_problems_extra(model, verbose, fLOG, extended_list) if extras is None or problems is None: # Not tested yet. continue # flat or not flat created, location_model, prefix_import, dest_pyspy_model = _handle_init_files( model, flat, location, verbose, dest_pyspy, fLOG) for init in created: yield init # loops on problems for prob in problems: if filter_exp is not None and not filter_exp(model, prob): continue (X_train, X_test, y_train, y_test, Xort_test, init_types, conv_options, method_name, output_index, dofit, predict_kwargs) = _get_problem_data(prob, None) for scenario_extra in extras: subset_problems = None optimisations = None new_conv_options = None if len(scenario_extra) > 2: options = scenario_extra[2] if isinstance(options, dict): subset_problems = options.get('subset_problems', None) optimisations = options.get('optim', None) new_conv_options = options.get('conv_options', None) else: subset_problems = options if subset_problems and isinstance(subset_problems, (list, set)): if prob not in subset_problems: # Skips unrelated problem for a specific configuration. continue elif subset_problems is not None: raise RuntimeError( # pragma: no cover "subset_problems must be a set or a list not {}.". format(subset_problems)) scenario, extra = scenario_extra[:2] if optimisations is None: optimisations = [None] if new_conv_options is None: new_conv_options = [{}] if (filter_scenario is not None and not filter_scenario( model, prob, scenario, extra, new_conv_options)): continue if verbose >= 3 and fLOG is not None: fLOG( "[create_asv_benchmark] model={} scenario={} optim={} extra={} dofit={} (problem={} method_name='{}')" .format(model.__name__, scenario, optimisations, extra, dofit, prob, method_name)) created = _create_asv_benchmark_file( location_model, opsets=opsets, model=model, scenario=scenario, optimisations=optimisations, extra=extra, dofit=dofit, problem=prob, runtime=runtime, new_conv_options=new_conv_options, X_train=X_train, X_test=X_test, y_train=y_train, y_test=y_test, Xort_test=Xort_test, init_types=init_types, conv_options=conv_options, method_name=method_name, dims=dims, n_features=n_features, output_index=output_index, predict_kwargs=predict_kwargs, exc=exc, prefix_import=prefix_import, execute=execute, location_pyspy=dest_pyspy_model, patterns=patterns) for cr in created: if cr in all_created: raise RuntimeError( "File '{}' was already created.".format(cr)) all_created.add(cr) if verbose > 1 and fLOG is not None: fLOG("[create_asv_benchmark] add '{}'.".format(cr)) yield cr
class _CommonAsvSklBenchmark: """ Common tests to all benchmarks testing converted :epkg:`scikit-learn` models. See `benchmark attributes <https://asv.readthedocs.io/en/stable/benchmarks.html#general>`_. """ # Part which changes. # params and param_names may be changed too. params = [ ['skl', 'pyrtc', 'ort'], # values for runtime [1, 10, 100, 10000, 100000], # values for N [4, 20], # values for nf [get_opset_number_from_onnx()], # values for opset ["float", "double"], # values for dtype [None], # values for optim ] param_names = ['rt', 'N', 'nf', 'opset', 'dtype', 'optim'] chk_method_name = None version = datetime.now().isoformat() pretty_source = "disabled" par_ydtype = numpy.int64 par_dofit = True par_convopts = None def _create_model(self): # pragma: no cover raise NotImplementedError("This method must be overwritten.") def _create_onnx_and_runtime(self, runtime, model, X, opset, dtype, optim): # pragma: no cover raise NotImplementedError("This method must be overwritten.") def _score_metric(self, X, y_exp, y_pred): # pragma: no cover raise NotImplementedError("This method must be overwritten.") def _optimize_onnx(self, onx): return onx def _get_xdtype(self, dtype): if dtype in ('float', numpy.float32): return numpy.float32 elif dtype in ('double', numpy.float64): return numpy.float64 raise ValueError( # pragma: no cover "Unknown dtype '{}'.".format(dtype)) def _get_dataset(self, nf, dtype): xdtype = self._get_xdtype(dtype) data = load_iris() X, y = data.data, data.target state = numpy.random.RandomState(seed=34) # pylint: disable=E1101 rnd = state.randn(*X.shape) / 3 X += rnd X = _modify_dimension(X, nf) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) Xt = X_test.astype(xdtype) yt = y_test.astype(self.par_ydtype) return (X_train, y_train), (Xt, yt) def _to_onnx(self, model, X, opset, dtype, optim): if optim is None or len(optim) == 0: options = self.par_convopts elif self.par_convopts and len(self.par_convopts) > 0: raise NotImplementedError( # pragma: no cover "Conflict between par_convopts={} and optim={}".format( self.par_convopts, optim)) else: # Expand common onnx options, see _nick_name_options. options = expand_onnx_options(model, optim) if dtype in (numpy.float64, 'double'): return to_onnx(model, X, options=options, target_opset=opset) return to_onnx(model, X, options=options, target_opset=opset) def _create_onnx_inference(self, onx, runtime): if 'onnxruntime' in runtime: old = onx.ir_version onx.ir_version = get_ir_version_from_onnx() else: old = None try: res = OnnxInference(onx, runtime=runtime) except RuntimeError as e: # pragma: no cover if "[ONNXRuntimeError]" in str(e): return RuntimeError("onnxruntime fails due to {}".format( str(e))) raise e if old is not None: onx.ir_version = old return res # Part which does not change. def runtime_name(self, runtime): """ Returns the runtime shortname. """ if runtime == 'skl': name = runtime elif runtime == 'ort': name = 'onnxruntime1' elif runtime == 'ort2': name = 'onnxruntime2' elif runtime == 'pyrt': name = 'python' elif runtime == 'pyrtc': name = 'python_compiled' else: raise ValueError( # pragma: no cover "Unknown runtime '{}'.".format(runtime)) return name def _name(self, nf, opset, dtype): last = 'cache-{}-nf{}-op{}-dt{}.pickle'.format(self.__class__.__name__, nf, opset, dtype) return last def setup_cache(self): "asv API" for dtype in self.params[4]: for opv in self.params[3]: for nf in self.params[2]: (X_train, y_train), (X, y) = self._get_dataset(nf, dtype) model = self._create_model() if self.par_dofit: set_random_state(model) model.fit(X_train, y_train) stored = {'model': model, 'X': X, 'y': y} filename = self._name(nf, opv, dtype) with open(filename, "wb") as f: pickle.dump(stored, f) if not os.path.exists(filename): raise RuntimeError( # pragma: no cover "Unable to dump model %r into %r." % (model, filename)) def setup(self, runtime, N, nf, opset, dtype, optim): "asv API" logger = getLogger('skl2onnx') logger.disabled = True register_converters() register_rewritten_operators() with open(self._name(nf, opset, dtype), "rb") as f: stored = pickle.load(f) self.stored = stored self.model = stored['model'] self.X, self.y = make_n_rows(stored['X'], N, stored['y']) onx, rt_, rt_fct_, rt_fct_track_ = self._create_onnx_and_runtime( runtime, self.model, self.X, opset, dtype, optim) self.onx = onx setattr(self, "rt_" + runtime, rt_) setattr(self, "rt_fct_" + runtime, rt_fct_) setattr(self, "rt_fct_track_" + runtime, rt_fct_track_) set_config(assume_finite=True) def time_predict(self, runtime, N, nf, opset, dtype, optim): "asv API" return getattr(self, "rt_fct_" + runtime)(self.X) def peakmem_predict(self, runtime, N, nf, opset, dtype, optim): "asv API" return getattr(self, "rt_fct_" + runtime)(self.X) def track_score(self, runtime, N, nf, opset, dtype, optim): "asv API" yp = getattr(self, "rt_fct_track_" + runtime)(self.X) return self._score_metric(self.X, self.y, yp) def track_onnxsize(self, runtime, N, nf, opset, dtype, optim): "asv API" return len(self.onx.SerializeToString()) def track_nbnodes(self, runtime, N, nf, opset, dtype, optim): "asv API" stats = onnx_statistics(self.onx) return stats.get('nnodes', 0) def track_vmlprodict(self, runtime, N, nf, opset, dtype, optim): "asv API" from mlprodict import __version__ return version2number(__version__) def track_vsklearn(self, runtime, N, nf, opset, dtype, optim): "asv API" from sklearn import __version__ return version2number(__version__) def track_vort(self, runtime, N, nf, opset, dtype, optim): "asv API" try: from onnxruntime import __version__ return version2number(__version__) except ImportError: # pragma: no cover return 0 def check_method_name(self, method_name): "Does some verifications. Fails if inconsistencies." if getattr(self, 'chk_method_name', None) not in (None, method_name): raise RuntimeError( # pragma: no cover "Method name must be '{}'.".format(method_name)) if getattr(self, 'chk_method_name', None) is None: raise RuntimeError( # pragma: no cover "Unable to check that the method name is correct " "(expected is '{}')".format(method_name))
def test_cpu_conv_group(self): x = numpy.random.rand(1, 3, 3, 4).astype(numpy.float32) W = numpy.random.rand(9, 1, 3, 3).astype(numpy.float32) onx = OnnxConv('X', 'W', output_names=['Y'], auto_pad='NOTSET', group=3, dilations=[1, 1], kernel_shape=[3, 3], strides=[1, 1], op_version=get_opset_number_from_onnx()) model_def = onx.to_onnx( { 'X': x.astype(numpy.float32), 'W': W.astype(numpy.float32) }, target_opset=get_opset_number_from_onnx()) oinf = OnnxInference(model_def) oinfrt = OnnxInference(model_def, runtime='onnxruntime1') d = oinf.sequence_[-1].ops_.atts_value self.assertIsInstance(d, dict) self.assertEqual(d['kernel_shape'].tolist(), [3, 3]) xs = [ numpy.random.rand(1, 3, 3, 4).astype(numpy.float32), numpy.array([ 1.0, 4.0, 7.0, 10.0, 13.0, 16.0, 19.0, 22.0, 25.0, 28.0, 31.0, 34.0, 2.0, 5.0, 8.0, 11.0, 14.0, 17.0, 20.0, 23.0, 26.0, 29.0, 32.0, 35.0, 3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0, 24.0, 27.0, 30.0, 33.0, 36.0 ], dtype=numpy.float32).reshape((1, 3, 3, 4)) ] Ws = [ numpy.random.rand(9, 1, 3, 3).astype(numpy.float32), numpy.array([ 1.0, 10.0, 19.0, 28.0, 37.0, 46.0, 55.0, 64.0, 73.0, 2.0, 11.0, 20.0, 29.0, 38.0, 47.0, 56.0, 65.0, 74.0, 3.0, 12.0, 21.0, 30.0, 39.0, 48.0, 57.0, 66.0, 75.0, 4.0, 13.0, 22.0, 31.0, 40.0, 49.0, 58.0, 67.0, 76.0, 5.0, 14.0, 23.0, 32.0, 41.0, 50.0, 59.0, 68.0, 77.0, 6.0, 15.0, 24.0, 33.0, 42.0, 51.0, 60.0, 69.0, 78.0, 7.0, 16.0, 25.0, 34.0, 43.0, 52.0, 61.0, 70.0, 79.0, 8.0, 17.0, 26.0, 35.0, 44.0, 53.0, 62.0, 71.0, 80.0, 9.0, 18.0, 27.0, 36.0, 45.0, 54.0, 63.0, 72.0, 81.0 ], dtype=numpy.float32).reshape((9, 1, 3, 3)) ] for x, W in zip(xs, Ws): x = numpy.asfortranarray(x) W = numpy.asfortranarray(W) got = oinf.run({'X': x, 'W': W}) gotrt = oinfrt.run({'X': x, 'W': W}) diff = list(numpy.abs((gotrt['Y'] - got['Y']).ravel())) sdiff = list(sorted(diff)) if sdiff[-1] > 1e-5: raise AssertionError("runtimes disagree {}".format(sdiff[-5:])) for ii in range(len(diff)): # pylint: disable=C0200 if numpy.isnan(diff[ii]): raise AssertionError( "runtimes disagree about nan {}: {} # {} ? {}".format( ii, diff[ii], gotrt['Y'].ravel()[ii], got['Y'].ravel()[ii])) self.assertEqualArray(gotrt['Y'], got['Y'], decimal=5)
def onnx_test_knn_single_classreg(self, dtype, n_targets=1, debug=False, add_noise=False, runtime='python', target_opset=None, optim=None, kind='reg', level=1, largest0=True, metric_params=None, **kwargs): iris = load_iris() X, y = iris.data, iris.target if add_noise: X += numpy.random.randn(X.shape[0], X.shape[1]) * 10 if kind == 'reg': y = y.astype(dtype) elif kind == 'bin': y = (y % 2).astype(numpy.int64) elif kind == 'mcl': y = y.astype(numpy.int64) else: raise AssertionError("unknown '{}'".format(kind)) if n_targets != 1: yn = numpy.empty((y.shape[0], n_targets), dtype=dtype) for i in range(n_targets): yn[:, i] = y + i y = yn X_train, X_test, y_train, _ = train_test_split(X, y, random_state=11) X_test = X_test.astype(dtype) if kind in ('bin', 'mcl'): clr = KNeighborsClassifier(metric_params=metric_params, **kwargs) elif kind == 'reg': clr = KNeighborsRegressor(metric_params=metric_params, **kwargs) else: raise NotImplementedError(kind) clr.fit(X_train, y_train) if optim is None: options = None else: options = {clr.__class__: {'optim': 'cdist'}} if not largest0: if options is None: options = {} if clr.__class__ not in options: options[clr.__class__] = {} options[clr.__class__].update({'largest0': False}) if target_opset is None: opsets = list( sorted(set([9, 10, 11, 12, get_opset_number_from_onnx()]))) else: opsets = [target_opset] for ops in opsets: if ops is None: raise AssertionError("Cannot happen: {}.".format(opsets)) with self.subTest(target_opset=ops): try: model_def = to_onnx(clr, X_train.astype(dtype), rewrite_ops=True, target_opset=ops, options=options) except NameError as e: if "Option 'largest0' not in" in str(e): continue if 'onnxruntime' in runtime: model_def.ir_version = get_ir_version_from_onnx() try: if runtime == 'onnxruntime2': oinf = _capture_output( lambda: OnnxInference(model_def, runtime=runtime), # pylint: disable=W0640 'c')[0] else: oinf = OnnxInference(model_def, runtime=runtime) except (RuntimeError, TypeError, OrtInvalidArgument) as e: if "No Op registered for Identity with domain_version of 12" in str( e): continue if debug: raise AssertionError( "Unable to create a model for target_opset={}\n----\n{}\n----" .format(ops, str(model_def)[:100])) from e if "Unknown model file format version." in str(e): continue raise AssertionError( "Unable to create model for opset={} and runtime='{}'\n{}" "".format(ops, runtime, str(model_def)[:100])) from e if debug: y = oinf.run({'X': X_test}, verbose=level, fLOG=print) else: y = oinf.run({'X': X_test}) lexp = clr.predict(X_test) if kind == 'reg': self.assertEqual(list(sorted(y)), ['variable']) if dtype == numpy.float32: self.assertEqualArray(lexp, y['variable'], decimal=5, squeeze=True) else: self.assertEqualArray(lexp, y['variable'], squeeze=True) else: self.assertEqual(list(sorted(y)), ['output_label', 'output_probability']) self.assertEqualArray(lexp, y['output_label']) lprob = clr.predict_proba(X_test) self.assertEqualArray(lprob, DataFrame( y['output_probability']).values, decimal=5)
def create_asv_benchmark(location, opset_min=-1, opset_max=None, runtime=('scikit-learn', 'python_compiled'), models=None, skip_models=None, extended_list=True, dims=(1, 10, 100, 10000, 100000), n_features=(4, 20), dtype=None, verbose=0, fLOG=print, clean=True, conf_params=None, filter_exp=None, filter_scenario=None, flat=False, exc=False, build=None, execute=False, add_pyspy=False, env=None, matrix=None): """ Creates an :epkg:`asv` benchmark in a folder but does not run it. :param n_features: number of features to try :param dims: number of observations to try :param verbose: integer from 0 (None) to 2 (full verbose) :param opset_min: tries every conversion from this minimum opset, -1 to get the current opset defined by module :epkg:`onnx` :param opset_max: tries every conversion up to maximum opset, -1 to get the current opset defined by module :epkg:`onnx` :param runtime: runtime to check, *scikit-learn*, *python*, *python_compiled* compiles the graph structure and is more efficient when the number of observations is small, *onnxruntime1* to check :epkg:`onnxruntime`, *onnxruntime2* to check every ONNX node independently with onnxruntime, many runtime can be checked at the same time if the value is a comma separated list :param models: list of models to test or empty string to test them all :param skip_models: models to skip :param extended_list: extends the list of :epkg:`scikit-learn` converters with converters implemented in this module :param n_features: change the default number of features for a specific problem, it can also be a comma separated list :param dtype: '32' or '64' or None for both, limits the test to one specific number types :param fLOG: logging function :param clean: clean the folder first, otherwise overwrites the content :param conf_params: to overwrite some of the configuration parameters :param filter_exp: function which tells if the experiment must be run, None to run all, takes *model, problem* as an input :param filter_scenario: second function which tells if the experiment must be run, None to run all, takes *model, problem, scenario, extra* as an input :param flat: one folder for all files or subfolders :param exc: if False, raises warnings instead of exceptions whenever possible :param build: where to put the outputs :param execute: execute each script to make sure imports are correct :param add_pyspy: add an extra folder with code to profile each configuration :param env: None to use the default configuration or ``same`` to use the current one :param matrix: specifies versions for a module, example: ``{'onnxruntime': ['1.1.1', '1.1.2']}``, if a package name starts with `'~'`, the package is removed :return: created files The default configuration is the following: .. runpython:: :showcode: import pprint from mlprodict.asv_benchmark.create_asv import default_asv_conf pprint.pprint(default_asv_conf) The benchmark does not seem to work well with setting ``-environment existing:same``. The publishing fails. """ if opset_min == -1: opset_min = get_opset_number_from_onnx() if opset_max == -1: opset_max = get_opset_number_from_onnx() # pragma: no cover if verbose > 0 and fLOG is not None: # pragma: no cover fLOG("[create_asv_benchmark] opset in [{}, {}].".format( opset_min, opset_max)) # creates the folder if it does not exist. if not os.path.exists(location): if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] create folder '{}'.".format(location)) os.makedirs(location) location_test = os.path.join(location, 'benches') if not os.path.exists(location_test): if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] create folder '{}'.".format( location_test)) os.mkdir(location_test) # Cleans the content of the folder created = [] if clean: for name in os.listdir(location_test): full_name = os.path.join(location_test, name) if os.path.isfile(full_name): os.remove(full_name) # configuration conf = default_asv_conf.copy() if conf_params is not None: for k, v in conf_params.items(): conf[k] = v if build is not None: for fi in ['env_dir', 'results_dir', 'html_dir']: conf[fi] = os.path.join(build, conf[fi]) if env == 'same': if matrix is not None: raise ValueError("Parameter matrix must be None if env is 'same'.") conf['pythons'] = ['same'] conf['matrix'] = {} elif matrix is not None: drop_keys = set(p for p in matrix if p.startswith('~')) matrix = {k: v for k, v in matrix.items() if k not in drop_keys} conf['matrix'] = { k: v for k, v in conf['matrix'].items() if k not in drop_keys } conf['matrix'].update(matrix) elif env is not None: raise ValueError( # pragma: no cover "Unable to handle env='{}'.".format(env)) dest = os.path.join(location, "asv.conf.json") created.append(dest) with open(dest, "w", encoding='utf-8') as f: json.dump(conf, f, indent=4) if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] create 'asv.conf.json'.") # __init__.py dest = os.path.join(location, "__init__.py") with open(dest, "w", encoding='utf-8') as f: pass created.append(dest) if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] create '__init__.py'.") dest = os.path.join(location_test, '__init__.py') with open(dest, "w", encoding='utf-8') as f: pass created.append(dest) if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] create 'benches/__init__.py'.") # flask_server tool_dir = os.path.join(location, 'tools') if not os.path.exists(tool_dir): os.mkdir(tool_dir) fl = os.path.join(tool_dir, 'flask_serve.py') with open(fl, "w", encoding='utf-8') as f: f.write(flask_helper) if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] create 'flask_serve.py'.") # command line if sys.platform.startswith("win"): run_bash = os.path.join(tool_dir, 'run_asv.bat') # pragma: no cover else: run_bash = os.path.join(tool_dir, 'run_asv.sh') with open(run_bash, 'w') as f: f.write( textwrap.dedent(""" echo --BENCHRUN-- python -m asv run --show-stderr --config ./asv.conf.json echo --PUBLISH-- python -m asv publish --config ./asv.conf.json -o ./html echo --CSV-- python -m mlprodict asv2csv -f ./results -o ./data_bench.csv """)) # pyspy if add_pyspy: dest_pyspy = os.path.join(location, 'pyspy') if not os.path.exists(dest_pyspy): os.mkdir(dest_pyspy) else: dest_pyspy = None if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] create all tests.") created.extend( list( _enumerate_asv_benchmark_all_models( location_test, opset_min=opset_min, opset_max=opset_max, runtime=runtime, models=models, skip_models=skip_models, extended_list=extended_list, n_features=n_features, dtype=dtype, verbose=verbose, filter_exp=filter_exp, filter_scenario=filter_scenario, dims=dims, exc=exc, flat=flat, fLOG=fLOG, execute=execute, dest_pyspy=dest_pyspy))) if verbose > 0 and fLOG is not None: fLOG("[create_asv_benchmark] done.") return created