def _provider_name_to_device_type(provider_name):
if provider_name == 'CPUExecutionProvider':
return OrtDevice.cpu()
if provider_name == 'CUDAExecutionProvider': # pragma: no cover
return OrtDevice.cuda()
raise ValueError( # pragma: no cover
f'Unexpected provider name {provider_name!r}.')
def device_name(device):
"""
Returns the device name of a device.
:param device: OrtDevice
:return: string
"""
if device.device_type() == OrtDevice.cpu():
return 'Cpu'
if device.device_type() == OrtDevice.cuda(): # pragma: no cover
return 'Gpu'
raise RuntimeError( # pragma: no cover
f"Unexpected value for device type {device.device_type()!r}.")
def get_ort_device(device):
"""
Converts device into :epkg:`C_OrtDevice`.
:param device: any type
:return: :epkg:`C_OrtDevice`
Example:
::
get_ort_device('cpu')
get_ort_device('gpu')
get_ort_device('cuda')
get_ort_device('cuda:0')
"""
if isinstance(device, C_OrtDevice):
return device
if isinstance(device, str):
if device == 'cpu':
return C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(),
0)
if device in {'gpu', 'cuda:0', 'cuda', 'gpu:0'}:
return C_OrtDevice(C_OrtDevice.cuda(),
C_OrtDevice.default_memory(), 0)
if device.startswith('gpu:'):
idx = int(device[4:])
return C_OrtDevice(C_OrtDevice.cuda(),
C_OrtDevice.default_memory(), idx)
if device.startswith('cuda:'):
idx = int(device[5:])
return C_OrtDevice(C_OrtDevice.cuda(),
C_OrtDevice.default_memory(), idx)
raise ValueError( # pragma: no cover
"Unable to interpret string %r as a device." % device)
raise TypeError( # pragma: no cover
"Unable to interpret type %r, (%r) as de device." %
(type(device), device))
def get_ort_device_type(device):
"""
Converts device into device type.
:param device: string
:return: device type
"""
if isinstance(device, str):
if device == 'cuda':
return C_OrtDevice.cuda()
if device == 'cpu':
return C_OrtDevice.cpu()
raise ValueError( # pragma: no cover
f'Unsupported device type: {device!r}.')
if not hasattr(device, 'device_type'):
raise TypeError(f'Unsupported device type: {type(device)!r}.')
device_type = device.device_type()
if device_type in ('cuda', 1):
return C_OrtDevice.cuda()
if device_type in ('cpu', 0):
return C_OrtDevice.cpu()
raise ValueError( # pragma: no cover
f'Unsupported device type: {device_type!r}.')
def test_bind_input_types(self):
opset = onnx_opset_version()
devices = [(C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0), ['CPUExecutionProvider'])]
if "CUDAExecutionProvider" in onnxrt.get_all_providers():
devices.append((C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0), ['CUDAExecutionProvider']))
for device, provider in devices:
for dtype in [np.float32, np.float64, np.int32, np.uint32,
np.int64, np.uint64, np.int16, np.uint16,
np.int8, np.uint8, np.float16, np.bool_]:
with self.subTest(dtype=dtype, device=str(device)):
x = np.arange(8).reshape((-1, 2)).astype(dtype)
proto_dtype = NP_TYPE_TO_TENSOR_TYPE[x.dtype]
X = helper.make_tensor_value_info('X', proto_dtype, [None, x.shape[1]])
Y = helper.make_tensor_value_info('Y', proto_dtype, [None, x.shape[1]])
# inference
node_add = helper.make_node('Identity', ['X'], ['Y'])
# graph
graph_def = helper.make_graph([node_add], 'lr', [X], [Y], [])
model_def = helper.make_model(
graph_def, producer_name='dummy', ir_version=7,
producer_version="0",
opset_imports=[helper.make_operatorsetid('', opset)])
sess = onnxrt.InferenceSession(model_def.SerializeToString(), providers=provider)
bind = SessionIOBinding(sess._sess)
ort_value = C_OrtValue.ortvalue_from_numpy(x, device)
bind.bind_ortvalue_input('X', ort_value)
bind.bind_output('Y', device)
sess._sess.run_with_iobinding(bind, None)
ortvalue = bind.get_outputs()[0]
y = ortvalue.numpy()
assert_almost_equal(x, y)
bind = SessionIOBinding(sess._sess)
bind.bind_input('X', device, dtype, x.shape, ort_value.data_ptr())
bind.bind_output('Y', device)
sess._sess.run_with_iobinding(bind, None)
ortvalue = bind.get_outputs()[0]
y = ortvalue.numpy()
assert_almost_equal(x, y)
def ort_device_to_string(device):
"""
Returns a string representing the device.
Opposite of function @see fn get_ort_device.
:param device: see :epkg:`C_OrtDevice`
:return: string
"""
if not isinstance(device, C_OrtDevice):
raise TypeError(
f"device must be of type C_OrtDevice not {type(device)!r}.")
ty = device.device_type()
if ty == C_OrtDevice.cpu():
sty = 'cpu'
elif ty == C_OrtDevice.cuda():
sty = 'cuda'
else:
raise NotImplementedError( # pragma: no cover
f"Unable to guess device for {device!r} and type={ty!r}.")
idx = device.device_id()
if idx == 0:
return sty
return "%s:%d" % (sty, idx)
shape = inp.shape
res[name] = random_input(typ, shape, batch)
return res
#################################
# Profiling
# +++++++++
#
# Let's choose the device available on this machine.
# batch dimension is set to 10.
batch = 10
if get_device().upper() == 'GPU':
ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(),
0)
provider = 'CUDAExecutionProvider'
else:
ort_device = C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(),
0)
provider = 'CPUExecutionProvider'
print(f"provider = {provider!r}")
####################################
# We load the graph.
with open(filename, 'rb') as f:
onx = onnx.load(f)