def test_bind_input_types(self):
opset = onnx_opset_version()
devices = [(C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0), ['CPUExecutionProvider'])]
if "CUDAExecutionProvider" in onnxrt.get_all_providers():
devices.append((C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0), ['CUDAExecutionProvider']))
for device, provider in devices:
for dtype in [np.float32, np.float64, np.int32, np.uint32,
np.int64, np.uint64, np.int16, np.uint16,
np.int8, np.uint8, np.float16, np.bool_]:
with self.subTest(dtype=dtype, device=str(device)):
x = np.arange(8).reshape((-1, 2)).astype(dtype)
proto_dtype = NP_TYPE_TO_TENSOR_TYPE[x.dtype]
X = helper.make_tensor_value_info('X', proto_dtype, [None, x.shape[1]])
Y = helper.make_tensor_value_info('Y', proto_dtype, [None, x.shape[1]])
# inference
node_add = helper.make_node('Identity', ['X'], ['Y'])
# graph
graph_def = helper.make_graph([node_add], 'lr', [X], [Y], [])
model_def = helper.make_model(
graph_def, producer_name='dummy', ir_version=7,
producer_version="0",
opset_imports=[helper.make_operatorsetid('', opset)])
sess = onnxrt.InferenceSession(model_def.SerializeToString(), providers=provider)
bind = SessionIOBinding(sess._sess)
ort_value = C_OrtValue.ortvalue_from_numpy(x, device)
bind.bind_ortvalue_input('X', ort_value)
bind.bind_output('Y', device)
sess._sess.run_with_iobinding(bind, None)
ortvalue = bind.get_outputs()[0]
y = ortvalue.numpy()
assert_almost_equal(x, y)
bind = SessionIOBinding(sess._sess)
bind.bind_input('X', device, dtype, x.shape, ort_value.data_ptr())
bind.bind_output('Y', device)
sess._sess.run_with_iobinding(bind, None)
ortvalue = bind.get_outputs()[0]
y = ortvalue.numpy()
assert_almost_equal(x, y)
def get_ort_device(device):
"""
Converts device into :epkg:`C_OrtDevice`.
:param device: any type
:return: :epkg:`C_OrtDevice`
Example:
::
get_ort_device('cpu')
get_ort_device('gpu')
get_ort_device('cuda')
get_ort_device('cuda:0')
"""
if isinstance(device, C_OrtDevice):
return device
if isinstance(device, str):
if device == 'cpu':
return C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(),
0)
if device in {'gpu', 'cuda:0', 'cuda', 'gpu:0'}:
return C_OrtDevice(C_OrtDevice.cuda(),
C_OrtDevice.default_memory(), 0)
if device.startswith('gpu:'):
idx = int(device[4:])
return C_OrtDevice(C_OrtDevice.cuda(),
C_OrtDevice.default_memory(), idx)
if device.startswith('cuda:'):
idx = int(device[5:])
return C_OrtDevice(C_OrtDevice.cuda(),
C_OrtDevice.default_memory(), idx)
raise ValueError( # pragma: no cover
"Unable to interpret string %r as a device." % device)
raise TypeError( # pragma: no cover
"Unable to interpret type %r, (%r) as de device." %
(type(device), device))
def fcts_model(X, y, n_jobs):
"LinearRegression."
model = LinearRegression(n_jobs=n_jobs)
model.fit(X, y)
initial_types = [('X', FloatTensorType([None, X.shape[1]]))]
onx = to_onnx(model,
initial_types=initial_types,
black_op={'LinearRegressor'})
sess = InferenceSession(onx.SerializeToString(),
providers=['CPUExecutionProvider'])
outputs = [o.name for o in sess.get_outputs()]
oinf = OnnxInference(onx, runtime="python")
bind = SessionIOBinding(sess._sess)
# ort_device = C_OrtDevice.cpu()
ort_device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(),
0)
def predict_skl_predict(X, model=model):
return model.predict(X)
def predict_onnxrt_predict(X, sess=sess):
return sess.run(outputs[:1], {'X': X})[0]
def predict_onnx_inference(X, oinf=oinf):
return oinf.run({'X': X})["variable"]
def predict_onnxrt_predict_bind(X,
sess=sess,
bind=bind,
ort_device=ort_device):
if X.__array_interface__['strides'] is not None:
raise RuntimeError("onnxruntime only supports contiguous arrays.")
bind.bind_input('X', ort_device, X.dtype, X.shape,
X.__array_interface__['data'][0])
bind.bind_output('variable', ort_device)
sess._sess.run_with_iobinding(bind, None)
ortvalues = bind.get_outputs()
return ortvalues[0].numpy()
return {
'predict': {
'skl': predict_skl_predict,
'ort': predict_onnxrt_predict,
'numpy': predict_onnx_inference,
'ort-bind': predict_onnxrt_predict_bind
}
}
def benchmark(name,
onx,
fct_numpy,
*args,
dims=(1, 10, 100, 200, 500, 1000, 2000, 10000)):
sess = InferenceSession(onx.SerializeToString())
device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0)
names = [i.name for i in sess.get_inputs()]
out_names = [o.name for o in sess.get_outputs()]
if len(names) != len(args):
raise RuntimeError(f"Size mismatch {len(names)} != {len(args)}.")
rows = []
for dim in tqdm(dims):
new_args = [reshape(a, dim) for a in args]
ortvalues = [
C_OrtValue.ortvalue_from_numpy(a, device) for a in new_args
]
ms = measure_time(lambda: fct_numpy(*new_args), repeat=50, number=100)
ms.update(dict(name=name, impl='numpy', dim=dim))
rows.append(ms)
inps = {n: a for n, a in zip(names, new_args)}
ms = measure_time(lambda: sess.run(None, inps))
ms.update(dict(name=name, impl='sess', dim=dim))
rows.append(ms)
bind = SessionIOBinding(sess._sess)
ms = measure_time(lambda: bind_and_run(sess._sess, bind, names,
ortvalues, out_names, device))
ms.update(dict(name=name, impl='bind_run', dim=dim))
rows.append(ms)
ms = measure_time(lambda: nobind_just_run(sess._sess, bind))
ms.update(dict(name=name, impl='run', dim=dim))
rows.append(ms)
return rows
shape = inp.shape
res[name] = random_input(typ, shape, batch)
return res
#################################
# Profiling
# +++++++++
#
# Let's choose the device available on this machine.
# batch dimension is set to 10.
batch = 10
if get_device().upper() == 'GPU':
ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(),
0)
provider = 'CUDAExecutionProvider'
else:
ort_device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(),
0)
provider = 'CPUExecutionProvider'
print(f"provider = {provider!r}")
####################################
# We load the graph.
with open(filename, 'rb') as f:
onx = onnx.load(f)
def _create_onnx_graphs(self):
"""
Creates forward and backward ONNX graph.
The new class has the following attributes:
* `__doc__`: doc string
* `__module__`: module name (this file)
* `_run_options`: see :epkg:`RunOptions`
* `_sess`: :epkg:`InferenceSession` with the original graph
* `_sess_eval`: :epkg:`InferenceSession` on the graph
with weights as inputs
* `_training_agent`: :epkg:`TrainingAgent`
* `_cache`: :epkg:`OrtValueCache`
* `_logger`: logger
* `_input_names`: input names
* `_debug`: use debug mode
* `_grad_input_names`: gradient input names
* `_output_names`: output names
* `_weights_to_train`: names of the weights to train
Training attributes
* `_bw_fetches_names`: bw_fetches_names,
* `_fw_outputs_device_info`: fw_outputs_device_info,
* `_bw_outputs_device_info`: bw_outputs_device_info,
* `_fw_no_grad_output_device_info`: fw_no_grad_output_device_info,
* `_graph_info`: graph_info}
Additional attributes added if *keep_model* is True:
* `_trained_onnx`: ONNX graph for the gradient
* `_optimized_pre_grad_model`: evaluation ONNX graph taking
weights as inputs
* `_graph_builder`: :epkg:`OrtModuleGraphBuilder`
"""
logger = self._logger
if logger is not None:
logger.info("[OrtGradientForwardBackward] create training onnx")
logger.info("[OrtGradientForwardBackward] input_names=%r",
self.input_names)
logger.info("[OrtGradientForwardBackward] output_names=%r",
self.output_names)
logger.info("[OrtGradientForwardBackward] weights_to_train=%r",
self.weights_to_train)
builder = OrtModuleGraphBuilder()
if logger is not None:
cf = self.graph_builder_config.graph_transformer_config
cfp = cf.propagate_cast_ops_config
logger.info("[OrtGradientForwardBackward] "
"OrtModuleGraphBuilder.initialize")
logger.info(
"[OrtGradientForwardBackward] graph_builder_config=%s",
OrtGradientForwardBackward._repr_helper_(
self.graph_builder_config, indent=4))
logger.info(
"[OrtGradientForwardBackward] graph_builder_config."
"graph_transformer_config=%s",
OrtGradientForwardBackward._repr_helper_(cf, indent=4))
logger.info(
"[OrtGradientForwardBackward] graph_builder_config."
"graph_transformer_config.propagate_cast_ops_config=%s",
OrtGradientForwardBackward._repr_helper_(cfp, indent=4))
builder.initialize(self.onnx_model.SerializeToString(),
self.graph_builder_config)
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] OrtModuleGraphBuilder.build")
builder.build()
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] OrtModuleGraphBuilder.get_model")
train_onnx_model_serialized = builder.get_model()
optimized_pre_grad_model = builder.get_inference_optimized_model()
graph_info = builder.get_graph_info()
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] graph_info=%s",
OrtGradientForwardBackward._repr_helper_(graph_info, indent=4))
logger.info("[OrtGradientForwardBackward] create TrainSession")
logger.info(
"[OrtGradientForwardBackward] sess_options=%s",
OrtGradientForwardBackward._repr_helper_(self.sess_options,
indent=4))
logger.info("[OrtGradientForwardBackward] providers=%r",
self.providers)
sess = InferenceSession(train_onnx_model_serialized,
sess_options=self.sess_options,
provider_options=self.provider_options,
providers=self.providers)
if logger is not None:
logger.info("[OrtGradientForwardBackward] create InferenceSession")
sess_eval = InferenceSession(optimized_pre_grad_model,
sess_options=self.sess_options,
provider_options=self.provider_options,
providers=self.providers)
if logger is not None:
logger.info("[OrtGradientForwardBackward] create training agent")
grad_input_names = [obj.name for obj in sess.get_inputs()]
bw_fetches_names = [obj.name for obj in sess.get_outputs()]
fw_outputs_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(i),
OrtDevice.default_memory(), self.device_index)
for i in self.providers
]
bw_outputs_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(
self.providers[0]), OrtDevice.default_memory(),
self.device_index) for i in bw_fetches_names
]
fw_no_grad_output_device_info = [
OrtDevice(
OrtGradientForwardBackward._provider_name_to_device_type(
self.providers[0]), OrtDevice.default_memory(),
self.device_index) for i in self.output_names
]
try:
# onnxruntime>=1.12
training_agent = TrainingAgent(sess._sess, grad_input_names,
fw_outputs_device_info,
bw_fetches_names,
bw_outputs_device_info, 0)
except TypeError:
# onnxruntime<=1.11
training_agent = TrainingAgent(sess._sess, grad_input_names,
fw_outputs_device_info,
bw_fetches_names,
bw_outputs_device_info)
if logger is not None:
logger.info(
"[OrtGradientForwardBackward] instantiate dynamic class %r",
self.class_name)
logger.info("[OrtGradientForwardBackward] weights_to_train=%r",
self.weights_to_train)
logger.info("[OrtGradientForwardBackward] grad_input_names=%r",
grad_input_names)
logger.info("[OrtGradientForwardBackward] bw_fetches_names=%r",
bw_fetches_names)
logger.info("[OrtGradientForwardBackward] device_index=%r",
self.device_index)
devices = list(fw_outputs_device_info)
while len(devices) < len(grad_input_names):
devices.append(devices[-1])
trained_onnx = onnx.load(BytesIO(train_onnx_model_serialized))
onnx_loss = onnx.load(BytesIO(optimized_pre_grad_model))
for i, node in enumerate(trained_onnx.graph.node):
if node.name == '':
node.name = "N%d" % i
for i, node in enumerate(onnx_loss.graph.node):
if node.name == '':
node.name = "N%d" % i
kwargs = {
'_run_options': self.run_options,
'_sess': sess,
'_sess_eval': sess_eval,
'_training_agent': training_agent,
'_cache': OrtValueCache(),
'_logger': logger,
'_input_names': self.input_names,
'_grad_input_names': grad_input_names,
'_output_names': self.output_names,
'_bw_fetches_names': bw_fetches_names,
'_fw_outputs_device_info': fw_outputs_device_info,
'_bw_outputs_device_info': bw_outputs_device_info,
'_fw_no_grad_output_device_info': fw_no_grad_output_device_info,
'_weights_to_train': list(sorted(self.weights_to_train)),
'_graph_info': graph_info,
#
'_trained_onnx': trained_onnx,
'_optimized_pre_grad_model': onnx_loss,
'_graph_builder': builder,
'_devices': devices,
'_debug': self.debug
}
graph = kwargs['_trained_onnx'].graph
kwargs.update({
'_onx_inp': [o.name for o in graph.input],
'_onx_out': [o.name for o in graph.output]
})
if len(kwargs['_onx_inp']) != len(kwargs['_onx_out']):
raise RuntimeError( # pragma: no cover
"Gradient input and output are inconsistant: "
"%r != %r" % (kwargs['_onx_inp'], kwargs['_onx_out']))
return kwargs