def test_bind_input_types(self): opset = onnx_opset_version() devices = [(C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0), ['CPUExecutionProvider'])] if "CUDAExecutionProvider" in onnxrt.get_all_providers(): devices.append((C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0), ['CUDAExecutionProvider'])) for device, provider in devices: for dtype in [np.float32, np.float64, np.int32, np.uint32, np.int64, np.uint64, np.int16, np.uint16, np.int8, np.uint8, np.float16, np.bool_]: with self.subTest(dtype=dtype, device=str(device)): x = np.arange(8).reshape((-1, 2)).astype(dtype) proto_dtype = NP_TYPE_TO_TENSOR_TYPE[x.dtype] X = helper.make_tensor_value_info('X', proto_dtype, [None, x.shape[1]]) Y = helper.make_tensor_value_info('Y', proto_dtype, [None, x.shape[1]]) # inference node_add = helper.make_node('Identity', ['X'], ['Y']) # graph graph_def = helper.make_graph([node_add], 'lr', [X], [Y], []) model_def = helper.make_model( graph_def, producer_name='dummy', ir_version=7, producer_version="0", opset_imports=[helper.make_operatorsetid('', opset)]) sess = onnxrt.InferenceSession(model_def.SerializeToString(), providers=provider) bind = SessionIOBinding(sess._sess) ort_value = C_OrtValue.ortvalue_from_numpy(x, device) bind.bind_ortvalue_input('X', ort_value) bind.bind_output('Y', device) sess._sess.run_with_iobinding(bind, None) ortvalue = bind.get_outputs()[0] y = ortvalue.numpy() assert_almost_equal(x, y) bind = SessionIOBinding(sess._sess) bind.bind_input('X', device, dtype, x.shape, ort_value.data_ptr()) bind.bind_output('Y', device) sess._sess.run_with_iobinding(bind, None) ortvalue = bind.get_outputs()[0] y = ortvalue.numpy() assert_almost_equal(x, y)
def get_ort_device(device): """ Converts device into :epkg:`C_OrtDevice`. :param device: any type :return: :epkg:`C_OrtDevice` Example: :: get_ort_device('cpu') get_ort_device('gpu') get_ort_device('cuda') get_ort_device('cuda:0') """ if isinstance(device, C_OrtDevice): return device if isinstance(device, str): if device == 'cpu': return C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0) if device in {'gpu', 'cuda:0', 'cuda', 'gpu:0'}: return C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0) if device.startswith('gpu:'): idx = int(device[4:]) return C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), idx) if device.startswith('cuda:'): idx = int(device[5:]) return C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), idx) raise ValueError( # pragma: no cover "Unable to interpret string %r as a device." % device) raise TypeError( # pragma: no cover "Unable to interpret type %r, (%r) as de device." % (type(device), device))
def fcts_model(X, y, n_jobs): "LinearRegression." model = LinearRegression(n_jobs=n_jobs) model.fit(X, y) initial_types = [('X', FloatTensorType([None, X.shape[1]]))] onx = to_onnx(model, initial_types=initial_types, black_op={'LinearRegressor'}) sess = InferenceSession(onx.SerializeToString(), providers=['CPUExecutionProvider']) outputs = [o.name for o in sess.get_outputs()] oinf = OnnxInference(onx, runtime="python") bind = SessionIOBinding(sess._sess) # ort_device = C_OrtDevice.cpu() ort_device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0) def predict_skl_predict(X, model=model): return model.predict(X) def predict_onnxrt_predict(X, sess=sess): return sess.run(outputs[:1], {'X': X})[0] def predict_onnx_inference(X, oinf=oinf): return oinf.run({'X': X})["variable"] def predict_onnxrt_predict_bind(X, sess=sess, bind=bind, ort_device=ort_device): if X.__array_interface__['strides'] is not None: raise RuntimeError("onnxruntime only supports contiguous arrays.") bind.bind_input('X', ort_device, X.dtype, X.shape, X.__array_interface__['data'][0]) bind.bind_output('variable', ort_device) sess._sess.run_with_iobinding(bind, None) ortvalues = bind.get_outputs() return ortvalues[0].numpy() return { 'predict': { 'skl': predict_skl_predict, 'ort': predict_onnxrt_predict, 'numpy': predict_onnx_inference, 'ort-bind': predict_onnxrt_predict_bind } }
def benchmark(name, onx, fct_numpy, *args, dims=(1, 10, 100, 200, 500, 1000, 2000, 10000)): sess = InferenceSession(onx.SerializeToString()) device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0) names = [i.name for i in sess.get_inputs()] out_names = [o.name for o in sess.get_outputs()] if len(names) != len(args): raise RuntimeError(f"Size mismatch {len(names)} != {len(args)}.") rows = [] for dim in tqdm(dims): new_args = [reshape(a, dim) for a in args] ortvalues = [ C_OrtValue.ortvalue_from_numpy(a, device) for a in new_args ] ms = measure_time(lambda: fct_numpy(*new_args), repeat=50, number=100) ms.update(dict(name=name, impl='numpy', dim=dim)) rows.append(ms) inps = {n: a for n, a in zip(names, new_args)} ms = measure_time(lambda: sess.run(None, inps)) ms.update(dict(name=name, impl='sess', dim=dim)) rows.append(ms) bind = SessionIOBinding(sess._sess) ms = measure_time(lambda: bind_and_run(sess._sess, bind, names, ortvalues, out_names, device)) ms.update(dict(name=name, impl='bind_run', dim=dim)) rows.append(ms) ms = measure_time(lambda: nobind_just_run(sess._sess, bind)) ms.update(dict(name=name, impl='run', dim=dim)) rows.append(ms) return rows
shape = inp.shape res[name] = random_input(typ, shape, batch) return res ################################# # Profiling # +++++++++ # # Let's choose the device available on this machine. # batch dimension is set to 10. batch = 10 if get_device().upper() == 'GPU': ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0) provider = 'CUDAExecutionProvider' else: ort_device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0) provider = 'CPUExecutionProvider' print(f"provider = {provider!r}") #################################### # We load the graph. with open(filename, 'rb') as f: onx = onnx.load(f)
def _create_onnx_graphs(self): """ Creates forward and backward ONNX graph. The new class has the following attributes: * `__doc__`: doc string * `__module__`: module name (this file) * `_run_options`: see :epkg:`RunOptions` * `_sess`: :epkg:`InferenceSession` with the original graph * `_sess_eval`: :epkg:`InferenceSession` on the graph with weights as inputs * `_training_agent`: :epkg:`TrainingAgent` * `_cache`: :epkg:`OrtValueCache` * `_logger`: logger * `_input_names`: input names * `_debug`: use debug mode * `_grad_input_names`: gradient input names * `_output_names`: output names * `_weights_to_train`: names of the weights to train Training attributes * `_bw_fetches_names`: bw_fetches_names, * `_fw_outputs_device_info`: fw_outputs_device_info, * `_bw_outputs_device_info`: bw_outputs_device_info, * `_fw_no_grad_output_device_info`: fw_no_grad_output_device_info, * `_graph_info`: graph_info} Additional attributes added if *keep_model* is True: * `_trained_onnx`: ONNX graph for the gradient * `_optimized_pre_grad_model`: evaluation ONNX graph taking weights as inputs * `_graph_builder`: :epkg:`OrtModuleGraphBuilder` """ logger = self._logger if logger is not None: logger.info("[OrtGradientForwardBackward] create training onnx") logger.info("[OrtGradientForwardBackward] input_names=%r", self.input_names) logger.info("[OrtGradientForwardBackward] output_names=%r", self.output_names) logger.info("[OrtGradientForwardBackward] weights_to_train=%r", self.weights_to_train) builder = OrtModuleGraphBuilder() if logger is not None: cf = self.graph_builder_config.graph_transformer_config cfp = cf.propagate_cast_ops_config logger.info("[OrtGradientForwardBackward] " "OrtModuleGraphBuilder.initialize") logger.info( "[OrtGradientForwardBackward] graph_builder_config=%s", OrtGradientForwardBackward._repr_helper_( self.graph_builder_config, indent=4)) logger.info( "[OrtGradientForwardBackward] graph_builder_config." "graph_transformer_config=%s", OrtGradientForwardBackward._repr_helper_(cf, indent=4)) logger.info( "[OrtGradientForwardBackward] graph_builder_config." "graph_transformer_config.propagate_cast_ops_config=%s", OrtGradientForwardBackward._repr_helper_(cfp, indent=4)) builder.initialize(self.onnx_model.SerializeToString(), self.graph_builder_config) if logger is not None: logger.info( "[OrtGradientForwardBackward] OrtModuleGraphBuilder.build") builder.build() if logger is not None: logger.info( "[OrtGradientForwardBackward] OrtModuleGraphBuilder.get_model") train_onnx_model_serialized = builder.get_model() optimized_pre_grad_model = builder.get_inference_optimized_model() graph_info = builder.get_graph_info() if logger is not None: logger.info( "[OrtGradientForwardBackward] graph_info=%s", OrtGradientForwardBackward._repr_helper_(graph_info, indent=4)) logger.info("[OrtGradientForwardBackward] create TrainSession") logger.info( "[OrtGradientForwardBackward] sess_options=%s", OrtGradientForwardBackward._repr_helper_(self.sess_options, indent=4)) logger.info("[OrtGradientForwardBackward] providers=%r", self.providers) sess = InferenceSession(train_onnx_model_serialized, sess_options=self.sess_options, provider_options=self.provider_options, providers=self.providers) if logger is not None: logger.info("[OrtGradientForwardBackward] create InferenceSession") sess_eval = InferenceSession(optimized_pre_grad_model, sess_options=self.sess_options, provider_options=self.provider_options, providers=self.providers) if logger is not None: logger.info("[OrtGradientForwardBackward] create training agent") grad_input_names = [obj.name for obj in sess.get_inputs()] bw_fetches_names = [obj.name for obj in sess.get_outputs()] fw_outputs_device_info = [ OrtDevice( OrtGradientForwardBackward._provider_name_to_device_type(i), OrtDevice.default_memory(), self.device_index) for i in self.providers ] bw_outputs_device_info = [ OrtDevice( OrtGradientForwardBackward._provider_name_to_device_type( self.providers[0]), OrtDevice.default_memory(), self.device_index) for i in bw_fetches_names ] fw_no_grad_output_device_info = [ OrtDevice( OrtGradientForwardBackward._provider_name_to_device_type( self.providers[0]), OrtDevice.default_memory(), self.device_index) for i in self.output_names ] try: # onnxruntime>=1.12 training_agent = TrainingAgent(sess._sess, grad_input_names, fw_outputs_device_info, bw_fetches_names, bw_outputs_device_info, 0) except TypeError: # onnxruntime<=1.11 training_agent = TrainingAgent(sess._sess, grad_input_names, fw_outputs_device_info, bw_fetches_names, bw_outputs_device_info) if logger is not None: logger.info( "[OrtGradientForwardBackward] instantiate dynamic class %r", self.class_name) logger.info("[OrtGradientForwardBackward] weights_to_train=%r", self.weights_to_train) logger.info("[OrtGradientForwardBackward] grad_input_names=%r", grad_input_names) logger.info("[OrtGradientForwardBackward] bw_fetches_names=%r", bw_fetches_names) logger.info("[OrtGradientForwardBackward] device_index=%r", self.device_index) devices = list(fw_outputs_device_info) while len(devices) < len(grad_input_names): devices.append(devices[-1]) trained_onnx = onnx.load(BytesIO(train_onnx_model_serialized)) onnx_loss = onnx.load(BytesIO(optimized_pre_grad_model)) for i, node in enumerate(trained_onnx.graph.node): if node.name == '': node.name = "N%d" % i for i, node in enumerate(onnx_loss.graph.node): if node.name == '': node.name = "N%d" % i kwargs = { '_run_options': self.run_options, '_sess': sess, '_sess_eval': sess_eval, '_training_agent': training_agent, '_cache': OrtValueCache(), '_logger': logger, '_input_names': self.input_names, '_grad_input_names': grad_input_names, '_output_names': self.output_names, '_bw_fetches_names': bw_fetches_names, '_fw_outputs_device_info': fw_outputs_device_info, '_bw_outputs_device_info': bw_outputs_device_info, '_fw_no_grad_output_device_info': fw_no_grad_output_device_info, '_weights_to_train': list(sorted(self.weights_to_train)), '_graph_info': graph_info, # '_trained_onnx': trained_onnx, '_optimized_pre_grad_model': onnx_loss, '_graph_builder': builder, '_devices': devices, '_debug': self.debug } graph = kwargs['_trained_onnx'].graph kwargs.update({ '_onx_inp': [o.name for o in graph.input], '_onx_out': [o.name for o in graph.output] }) if len(kwargs['_onx_inp']) != len(kwargs['_onx_out']): raise RuntimeError( # pragma: no cover "Gradient input and output are inconsistant: " "%r != %r" % (kwargs['_onx_inp'], kwargs['_onx_out'])) return kwargs