def make_npy(self):
base = self.svm_num
samples, labels = [], []
paths = hp.get_paths(base)
for path in paths:
img = hp.get_image(base, path)
sample = self.__buddy_hog(img)
label = hp.get_name(path)
labels.append(label)
samples.append(sample)
samples = np.float32(samples)
labels = np.array(labels)
np.save(self.samples_num_file, samples)
np.save(self.labels_num_file, labels)
base = self.svm_sym
samples, labels = [], []
paths = hp.get_paths(base)
for path in paths:
img = hp.get_image(base, path)
sample = self.__buddy_hog(img)
label = hp.get_name(path) - 10
labels.append(label)
samples.append(sample)
samples = np.float32(samples)
labels = np.array(labels)
np.save(self.samples_sym_file, samples)
np.save(self.labels_sym_file, labels)
def __make_svm_from_images(self, dir_train, mode):
samples, labels = [], []
paths = hp.get_paths(dir_train)
modifier = 0
if mode == "num":
modifier = 0
elif mode == "sym":
modifier = 10
for path in paths:
img = hp.get_image(dir_train, path)
sample = self.__buddy_hog(img)
label = hp.get_name(path) - modifier
labels.append(label)
samples.append(sample)
samples = np.array(samples)
labels = np.array(labels)
rand = np.random.RandomState(321)
shuffle = rand.permutation(len(samples))
samples, labels = samples[shuffle], labels[shuffle]
samples = np.float32(samples)
model = SVM(C=2.67, gamma=5.383)
model.train(samples, labels)
return model
def __make_knn_from_images(self, mode):
base = ""
modifier = 0
samples, labels = [], []
if mode == "num":
base = self.knn_num
modifier = 0
elif mode == "sym":
base = self.knn_sym
modifier = 10
paths = hp.get_paths(base)
for path in paths:
img = hp.get_image(base, path)
sample = self.__buddy_hog(img)
label = hp.get_name(path) - modifier
labels.append(label)
samples.append(sample)
samples = np.float32(samples)
labels = np.array(labels)
model = KNearest(k=5)
model.train(samples, labels)
return model
def make_hog_num_file():
base = hp.small_test_num_images_15x20
samples, labels = [], []
paths = hp.get_paths(base)
for path in paths:
img = hp.get_image(base, path)
sample = buddy_hog(img)
label = hp.get_name(path)
labels.append(label)
samples.append(sample)
samples = np.float32(samples)
labels = np.array(labels)
labels = list(labels)
# head
document = ""
head1 = str(len(samples)) + " "
head2 = str(len(samples[0])) + " "
head3 = str("10") + "\n"
head = head1 + head2 + head3
document += head
for sample, label in zip(samples, labels):
input = ' '.join(map(str, list(sample)))
output = make_output_text_vector(int(label), 10)
document += input + "\n" + output
print document
with open(os.path.join("ann/test/", "test-hog-nums.dat"), 'wb') as temp_file:
temp_file.write(document)
def get_motename_from_ipv6(self):
mote_ipv6 = raw_input(">> Input Mote IPv6 :")
name = helper.get_name(mote_ipv6,args.workbook)
if name:
print("\n\nIPv6 : " + mote_ipv6 + " | Mote Name : " + name + "\n\n")
else:
print "\n\nName Not Found In Database \n\n"
self.detail_continue()
def testSessionOptionsAddFreeDimensionOverrideByDenotation(self):
so = onnxrt.SessionOptions()
so.add_free_dimension_override_by_denotation("DATA_BATCH", 3)
so.add_free_dimension_override_by_denotation("DATA_CHANNEL", 5)
sess = onnxrt.InferenceSession(get_name("abs_free_dimensions.onnx"),
so)
input_name = sess.get_inputs()[0].name
self.assertEqual(input_name, "x")
input_shape = sess.get_inputs()[0].shape
# Free dims with denotations - "DATA_BATCH" and "DATA_CHANNEL" have values assigned to them.
self.assertEqual(input_shape, [3, 5, 5])
def testListAsInput(self):
sess = onnxrt.InferenceSession(get_name("mul_1.onnx"))
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
input_name = sess.get_inputs()[0].name
res = sess.run([], {input_name: x.tolist()})
output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]],
dtype=np.float32)
np.testing.assert_allclose(output_expected,
res[0],
rtol=1e-05,
atol=1e-08)
def testSessionOptionsAddFreeDimensionOverrideByName(self):
so = onnxrt.SessionOptions()
so.add_free_dimension_override_by_name("Dim1", 4)
so.add_free_dimension_override_by_name("Dim2", 6)
sess = onnxrt.InferenceSession(get_name("abs_free_dimensions.onnx"),
so)
input_name = sess.get_inputs()[0].name
self.assertEqual(input_name, "x")
input_shape = sess.get_inputs()[0].shape
# "Dim1" and "Dim2" have values assigned to them.
self.assertEqual(input_shape, [4, 6, 5])
def testRunModel(self):
name = get_name("mul_1.onnx")
rep = backend.prepare(name)
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
res = rep.run(x)
output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]],
dtype=np.float32)
np.testing.assert_allclose(output_expected,
res[0],
rtol=1e-05,
atol=1e-08)
def test_run_model_mlnet(self):
available_providers = onnxrt.get_available_providers()
# The Windows GPU CI pipeline builds the wheel with both CUDA and DML enabled and ORT does not support cases
# where one node is assigned to CUDA and one node to DML, as it doesn't have the data transfer capabilities to
# deal with potentially different device memory. Hence, use a session with only DML and CPU (excluding CUDA)
# for this test as it breaks with both CUDA and DML registered.
if "CUDAExecutionProvider" in available_providers and "DmlExecutionProvider" in available_providers:
sess = onnxrt.InferenceSession(
get_name("mlnet_encoder.onnx"),
None,
["DmlExecutionProvider", "CPUExecutionProvider"],
)
else:
sess = onnxrt.InferenceSession(get_name("mlnet_encoder.onnx"), providers=available_providers)
names = [_.name for _ in sess.get_outputs()]
self.assertEqual(["C00", "C12"], names)
c0 = np.array([5.0], dtype=np.float32).reshape(1, 1)
c1 = np.array([b"A\0A\0", b"B\0B\0", b"C\0C\0"], np.void).reshape(1, 3)
res = sess.run(None, {"C0": c0, "C1": c1})
mat = res[1]
total = mat.sum()
self.assertEqual(total, 2)
self.assertEqual(
list(mat.ravel()),
list(np.array([[[0.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]]]).ravel()),
)
# In memory, the size of each element is fixed and equal to the
# longest element. We cannot use bytes because numpy is trimming
# every final 0 for strings and bytes before creating the array
# (to save space). It does not have this behaviour for void
# but as a result, numpy does not know anymore the size
# of each element, they all have the same size.
c1 = np.array([b"A\0A\0\0", b"B\0B\0\0", b"C\0C\0\0"], np.void).reshape(1, 3)
res = sess.run(None, {"C0": c0, "C1": c1})
mat = res[1]
total = mat.sum()
self.assertEqual(total, 0)
def test_loop_to_scan(self):
loop_model_filename = get_name(
"nuphar_tiny_model_with_loop_shape_infered.onnx")
scan_model_filename = "nuphar_tiny_model_with_loop_shape_infered_converted_to_scan.onnx"
subprocess.run([
sys.executable, '-m', 'onnxruntime.nuphar.model_editor', '--input',
loop_model_filename, '--output', scan_model_filename, '--mode',
'loop_to_scan'
],
check=True)
validate_with_ort(loop_model_filename, scan_model_filename)
def runBaseTest1():
sess = onnxrt.InferenceSession(get_name("mul_1.onnx"))
self.assertTrue('CUDAExecutionProvider' in sess.get_providers())
option1 = {'device_id': 0}
sess.set_providers(['CUDAExecutionProvider'], [option1])
self.assertEqual(['CUDAExecutionProvider', 'CPUExecutionProvider'], sess.get_providers())
option2 = {'device_id': -1}
with self.assertRaises(RuntimeError):
sess.set_providers(['CUDAExecutionProvider'], [option2])
sess.set_providers(['CUDAExecutionProvider', 'CPUExecutionProvider'], [option1, {}])
self.assertEqual(['CUDAExecutionProvider', 'CPUExecutionProvider'], sess.get_providers())
def test_loop_to_scan_tool(self):
loop_model_filename = get_name("nuphar_tiny_model_with_loop_shape_infered.onnx")
scan_model_filename = "nuphar_tiny_model_with_loop_shape_infered_converted_to_scan.onnx"
subprocess.run([
sys.executable, '-m', 'onnxruntime.nuphar.model_tools',
'--input', loop_model_filename,
'--output', scan_model_filename,
'--tool', 'convert_loop_to_scan_and_validate',
'--symbolic_dims', 'sequence=30'
], check=True)
validate_with_ort(loop_model_filename, scan_model_filename)
def test_loop_to_scan_with_inconvertible_loop(self):
# nuphar_onnx_test_loop11_inconvertible_loop.onnx contains a Loop op with dynamic loop count.
# This Loop op cannot be converted to a Scan op.
# Set --keep_unconvertible_loop_ops option so conversion will not fail due to unconvertible loop ops.
loop_model_filename = get_name("nuphar_onnx_test_loop11_inconvertible_loop.onnx")
scan_model_filename = "nuphar_onnx_test_loop11_inconvertible_loop_unchanged.onnx"
subprocess.run([
sys.executable, '-m', 'onnxruntime.nuphar.model_editor',
'--input', loop_model_filename,
'--output', scan_model_filename, '--mode', 'loop_to_scan',
'--keep_unconvertible_loop_ops'
], check=True)
def test_bind_input_and_non_preallocated_output(self):
session = onnxrt.InferenceSession(get_name("mul_1.onnx"), providers=onnxrt.get_available_providers())
io_binding = session.io_binding()
# Bind input to CUDA
io_binding.bind_input('X', 'cuda', 0, np.float32, [3, 2], self.create_ortvalue_input_on_gpu().data_ptr())
# Bind output to CUDA
io_binding.bind_output('Y', 'cuda')
# Sync if different CUDA streams
io_binding.synchronize_inputs()
# Invoke Run
session.run_with_iobinding(io_binding)
# Sync if different CUDA streams
io_binding.synchronize_outputs()
# This call returns an OrtValue which has data allocated by ORT on CUDA
ort_outputs = io_binding.get_outputs()
self.assertEqual(len(ort_outputs), 1)
self.assertEqual(ort_outputs[0].device_name(), "cuda")
# Validate results (by copying results to CPU by creating a Numpy object)
self.assertTrue(np.array_equal(self.create_expected_output(), ort_outputs[0].numpy()))
# We should be able to repeat the above process as many times as we want - try once more
ort_outputs = io_binding.get_outputs()
self.assertEqual(len(ort_outputs), 1)
self.assertEqual(ort_outputs[0].device_name(), "cuda")
# Validate results (by copying results to CPU by creating a Numpy object)
self.assertTrue(np.array_equal(self.create_expected_output(), ort_outputs[0].numpy()))
# Change the bound input and validate the results in the same bound OrtValue
# Bind alternate input to CUDA
io_binding.bind_input('X', 'cuda', 0, np.float32, [3, 2], self.create_ortvalue_alternate_input_on_gpu().data_ptr())
# Sync if different CUDA streams
io_binding.synchronize_inputs()
# Invoke Run
session.run_with_iobinding(io_binding)
# Sync if different CUDA streams
io_binding.synchronize_outputs()
# This call returns an OrtValue which has data allocated by ORT on CUDA
ort_outputs = io_binding.get_outputs()
self.assertEqual(len(ort_outputs), 1)
self.assertEqual(ort_outputs[0].device_name(), "cuda")
# Validate results (by copying results to CPU by creating a Numpy object)
self.assertTrue(np.array_equal(self.create_expected_output_alternate(), ort_outputs[0].numpy()))
def testRunSparseOutputOrtValueVector(self):
"""
Try running models using the new run_with_ort_values
sparse_initializer_as_output.onnx - requires no inputs, but only one output
that comes from the initializer
"""
# The below values are a part of the model
sess = onnxrt.InferenceSession(
get_name("sparse_initializer_as_output.onnx"),
providers=onnxrt.get_available_providers(),
)
res = sess._sess.run_with_ort_values({}, ["values"], RunOptions())
self.assertIsInstance(res, OrtValueVector)
def testModelSerialization(self):
try:
so = onnxrt.SessionOptions()
so.log_verbosity_level = 1
so.logid = "TestModelSerialization"
so.optimized_model_filepath = "./PythonApiTestOptimizedModel.onnx"
onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=so)
self.assertTrue(os.path.isfile(so.optimized_model_filepath))
except Fail as onnxruntime_error:
if str(onnxruntime_error) == "[ONNXRuntimeError] : 1 : FAIL : Unable to serialize model as it contains" \
" compiled nodes. Please disable any execution providers which generate compiled nodes.":
pass
else:
raise onnxruntime_error
def testRunModel2(self):
sess = onnxrt.InferenceSession(get_name("matmul_1.onnx"))
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
input_name = sess.get_inputs()[0].name
self.assertEqual(input_name, "X")
input_shape = sess.get_inputs()[0].shape
self.assertEqual(input_shape, [3, 2])
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "Y")
output_shape = sess.get_outputs()[0].shape
self.assertEqual(output_shape, [3, 1])
res = sess.run([output_name], {input_name: x})
output_expected = np.array([[5.0], [11.0], [17.0]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
def testGraphOptimizationLevel(self):
opt = onnxrt.SessionOptions()
# default should be all optimizations optimization
self.assertEqual(opt.graph_optimization_level,
onnxrt.GraphOptimizationLevel.ORT_ENABLE_ALL)
opt.graph_optimization_level = onnxrt.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
self.assertEqual(opt.graph_optimization_level,
onnxrt.GraphOptimizationLevel.ORT_ENABLE_EXTENDED)
sess = onnxrt.InferenceSession(get_name("logicaland.onnx"),
sess_options=opt)
a = np.array([[True, True], [False, False]], dtype=np.bool)
b = np.array([[True, False], [True, False]], dtype=np.bool)
res = sess.run([], {'input1:0': a, 'input:0': b})
def testDictVectorizer(self):
sess = onnxrt.InferenceSession(
get_name("pipeline_vectorize.onnx"),
providers=onnxrt.get_available_providers(),
)
input_name = sess.get_inputs()[0].name
self.assertEqual(input_name, "float_input")
input_type = str(sess.get_inputs()[0].type)
self.assertEqual(input_type, "map(int64,tensor(float))")
input_shape = sess.get_inputs()[0].shape
self.assertEqual(input_shape, [])
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "variable1")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, "tensor(float)")
output_shape = sess.get_outputs()[0].shape
self.assertEqual(output_shape, [1, 1])
# Python type
x = {0: 25.0, 1: 5.13, 2: 0.0, 3: 0.453, 4: 5.966}
res = sess.run([output_name], {input_name: x})
output_expected = np.array([[49.752754]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
xwrong = x.copy()
xwrong["a"] = 5.6
try:
res = sess.run([output_name], {input_name: xwrong})
except RuntimeError as e:
self.assertIn(
"Unexpected key type <class 'str'>, it cannot be linked to C type int64_t",
str(e),
)
# numpy type
x = {np.int64(k): np.float32(v) for k, v in x.items()}
res = sess.run([output_name], {input_name: x})
output_expected = np.array([[49.752754]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
x = {np.int64(k): np.float64(v) for k, v in x.items()}
res = sess.run([output_name], {input_name: x})
output_expected = np.array([[49.752754]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
x = {np.int32(k): np.float64(v) for k, v in x.items()}
res = sess.run([output_name], {input_name: x})
output_expected = np.array([[49.752754]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
def testRunModelMultipleThreads(self):
so = onnxrt.SessionOptions()
so.log_verbosity_level = 1
so.logid = "MultiThreadsTest"
sess = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=so)
ro1 = onnxrt.RunOptions()
ro1.logid = "thread1"
t1 = threading.Thread(target=self.run_model, args=(sess, ro1))
ro2 = onnxrt.RunOptions()
ro2.logid = "thread2"
t2 = threading.Thread(target=self.run_model, args=(sess, ro2))
t1.start()
t2.start()
t1.join()
t2.join()
def testSessionOptionsAddInitializer(self):
# Create an initializer and add it to a SessionOptions instance
so = onnxrt.SessionOptions()
# This initializer is different from the actual initializer in the model for "W"
ortvalue_initializer = onnxrt.OrtValue.ortvalue_from_numpy(np.array([[2.0, 1.0], [4.0, 3.0], [6.0, 5.0]], dtype=np.float32))
# The user should manage the life cycle of this OrtValue and should keep it in scope
# as long as any session that is going to be reliant on it is in scope
so.add_initializer("W", ortvalue_initializer)
# Create an InferenceSession that only uses the CPU EP and validate that it uses the
# initializer provided via the SessionOptions instance (overriding the model initializer)
# We only use the CPU EP because the initializer we created is on CPU and we want the model to use that
sess = onnxrt.InferenceSession(get_name("mul_1.onnx"), so, ['CPUExecutionProvider'])
res = sess.run(["Y"], {"X": np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)})
self.assertTrue(np.array_equal(res[0], np.array([[2.0, 2.0], [12.0, 12.0], [30.0, 30.0]], dtype=np.float32)))
def testRunModelWithCudaGraph(self):
if 'CUDAExecutionProvider' in onnxrt.get_available_providers():
providers = [('CUDAExecutionProvider', {
'enable_cuda_graph': True
})]
INPUT_SIZE = 1280
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]] * INPUT_SIZE,
dtype=np.float32)
y = np.array([[0.0], [0.0], [0.0]] * INPUT_SIZE, dtype=np.float32)
x_ortvalue = onnxrt.OrtValue.ortvalue_from_numpy(x, 'cuda', 0)
y_ortvalue = onnxrt.OrtValue.ortvalue_from_numpy(y, 'cuda', 0)
session = onnxrt.InferenceSession(get_name("matmul_2.onnx"),
providers=providers)
io_binding = session.io_binding()
# Bind the input and output
io_binding.bind_ortvalue_input('X', x_ortvalue)
io_binding.bind_ortvalue_output('Y', y_ortvalue)
# One regular run for the necessary memory allocation and cuda graph capturing
session.run_with_iobinding(io_binding)
expected_y = np.array([[5.0], [11.0], [17.0]] * INPUT_SIZE,
dtype=np.float32)
np.testing.assert_allclose(expected_y,
y_ortvalue.numpy(),
rtol=1e-05,
atol=1e-05)
# After capturing, CUDA graph replay happens from this Run onwards
session.run_with_iobinding(io_binding)
np.testing.assert_allclose(expected_y,
y_ortvalue.numpy(),
rtol=1e-05,
atol=1e-05)
# Update input and then replay CUDA graph
x_ortvalue.update_inplace(
np.array([[10.0, 20.0], [30.0, 40.0], [50.0, 60.0]] *
INPUT_SIZE,
dtype=np.float32))
session.run_with_iobinding(io_binding)
np.testing.assert_allclose(np.array([[50.0], [110.0], [170.0]] *
INPUT_SIZE,
dtype=np.float32),
y_ortvalue.numpy(),
rtol=1e-05,
atol=1e-05)
def testBertCheckpointingLoadZero(self):
return # disable flaky test temporarily
torch.manual_seed(1)
onnxruntime.set_seed(1)
model, _, device = create_ort_trainer(gradient_accumulation_steps=1,
use_mixed_precision=False,
allreduce_post_accumulation=True,
use_simple_model_desc=True,
loss_scaler=None)
ckpt_dir = get_name("ort_ckpt")
load_checkpoint(model, ckpt_dir, 'bert_toy_lamb')
expected_eval_loss = [10.997552871]
input_ids = torch.tensor(
[[26598], [21379], [19922], [5219], [5644], [20559], [23777],
[25672], [22969], [16824], [16822], [635], [27399], [20647],
[18519], [15546]],
device=device)
segment_ids = torch.tensor([[0], [1], [0], [1], [0], [0], [1], [0],
[0], [1], [1], [0], [0], [1], [1], [1]],
device=device)
input_mask = torch.tensor([[0], [0], [0], [0], [1], [1], [1], [0], [1],
[1], [0], [0], [0], [1], [0], [0]],
device=device)
masked_lm_labels = torch.tensor(
[[25496], [16184], [11005], [16228], [14884], [21660], [8678],
[23083], [4027], [8397], [11921], [1333], [26482], [1666],
[17925], [27978]],
device=device)
next_sentence_labels = torch.tensor(
[0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0], device=device)
actual_eval_loss = model.eval_step(input_ids,
segment_ids,
input_mask,
masked_lm_labels,
next_sentence_labels,
fetches=['loss'])
actual_eval_loss = actual_eval_loss.cpu().numpy().item(0)
# import pdb; pdb.set_trace()
print(actual_eval_loss)
rtol = 1e-03
assert_allclose(expected_eval_loss,
actual_eval_loss,
err_msg="evaluation loss mismatch")
def runBaseTest2():
sess = onnxrt.InferenceSession(get_name("mul_1.onnx"))
self.assertTrue('CUDAExecutionProvider' in sess.get_providers())
# test get/set of "cuda_mem_limit" configuration.
options = sess.get_provider_options()
self.assertTrue('CUDAExecutionProvider' in options)
option = options['CUDAExecutionProvider']
self.assertTrue('cuda_mem_limit' in option)
ori_mem_limit = option['cuda_mem_limit']
new_mem_limit = int(ori_mem_limit) // 2
option['cuda_mem_limit'] = new_mem_limit
sess.set_providers(['CUDAExecutionProvider'], [option])
options = sess.get_provider_options()
self.assertEqual(options['CUDAExecutionProvider']['cuda_mem_limit'], str(new_mem_limit))
option['cuda_mem_limit'] = ori_mem_limit
sess.set_providers(['CUDAExecutionProvider'], [option])
options = sess.get_provider_options()
self.assertEqual(options['CUDAExecutionProvider']['cuda_mem_limit'], ori_mem_limit)
option['cuda_mem_limit'] = -1024
with self.assertRaises(RuntimeError):
sess.set_providers(['CUDAExecutionProvider'], [option])
option['cuda_mem_limit'] = 1024.1024
with self.assertRaises(RuntimeError):
sess.set_providers(['CUDAExecutionProvider'], [option])
option['cuda_mem_limit'] = 'wrong_value'
with self.assertRaises(RuntimeError):
sess.set_providers(['CUDAExecutionProvider'], [option])
# test get/set of "arena_extend_strategy" configuration.
options = sess.get_provider_options()
self.assertTrue('CUDAExecutionProvider' in options)
option = options['CUDAExecutionProvider']
self.assertTrue('arena_extend_strategy' in option)
for strategy in ['kNextPowerOfTwo', 'kSameAsRequested']:
option['arena_extend_strategy'] = strategy
sess.set_providers(['CUDAExecutionProvider'], [option])
options = sess.get_provider_options()
self.assertEqual(options['CUDAExecutionProvider']['arena_extend_strategy'], strategy)
option['arena_extend_strategy'] = 'wrong_value'
with self.assertRaises(RuntimeError):
sess.set_providers(['CUDAExecutionProvider'], [option])
def testProfilerWithSessionOptions(self):
so = onnxrt.SessionOptions()
so.enable_profiling = True
sess = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=so)
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
sess.run([], {'X': x})
profile_file = sess.end_profiling()
tags = ['pid', 'dur', 'ts', 'ph', 'X', 'name', 'args']
with open(profile_file) as f:
lines = f.readlines()
self.assertTrue('[' in lines[0])
for i in range(1, 8):
for tag in tags:
self.assertTrue(tag in lines[i])
self.assertTrue(']' in lines[8])
def testRunModelFromBytes(self):
with open(get_name("mul_1.onnx"), "rb") as f:
content = f.read()
sess = onnxrt.InferenceSession(content)
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
input_name = sess.get_inputs()[0].name
self.assertEqual(input_name, "X")
input_shape = sess.get_inputs()[0].shape
self.assertEqual(input_shape, [3, 2])
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "Y")
output_shape = sess.get_outputs()[0].shape
self.assertEqual(output_shape, [3, 2])
res = sess.run([output_name], {input_name: x})
output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
def testZipMapInt64Float(self):
sess = onnxrt.InferenceSession(get_name("zipmap_int64float.onnx"))
x = np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3))
x_name = sess.get_inputs()[0].name
self.assertEqual(x_name, "X")
x_type = sess.get_inputs()[0].type
self.assertEqual(x_type, 'tensor(float)')
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "Z")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, 'seq(map(int64,tensor(float)))')
output_expected = [{10: 1.0, 20: 0.0, 30: 3.0}, {10: 44.0, 20: 23.0, 30: 11.0}]
res = sess.run([output_name], {x_name: x})
self.assertEqual(output_expected, res[0])
def testMNISTResumeTrainingAndTesting(self):
torch.manual_seed(1)
device = torch.device("cuda")
mnist = MNISTWrapper()
train_loader, test_loader = mnist.get_loaders()
model, model_desc = mnist.get_model()
learningRate = 0.01
args_epochs = 2
args_checkpoint_epoch = 1
# should match those in test without checkpointing
expected_losses = [0.26509523391723633, 0.24135658144950867, 0.2397943139076233, 0.3351520597934723,
0.20998981595039368, 0.31488314270973206, 0.18481917679309845, 0.34727591276168823,
0.2971782684326172, 0.3609251379966736]
expected_test_losses = [0.25632242965698243]
expected_test_accuracies = [0.9264]
actual_losses = []
actual_test_losses, actual_accuracies = [], []
# restore from checkpoint
resume_trainer = mnist.get_trainer(model, model_desc, device)
checkpoint = torch.load(get_name("ckpt_mnist.pt"), map_location="cpu")
torch.set_rng_state(checkpoint['rng_state'])
resume_trainer.load_state_dict(checkpoint['model'], strict=True)
# continue ..
for epoch in range(args_checkpoint_epoch + 1, args_epochs + 1):
actual_losses = [*actual_losses, *mnist.train_with_trainer(learningRate, resume_trainer, device, train_loader, epoch)]
test_loss, accuracy = mnist.test_with_trainer(resume_trainer, device, test_loader)
actual_test_losses = [*actual_test_losses, test_loss]
actual_accuracies = [*actual_accuracies, accuracy]
print("actual_losses=", actual_losses)
print("actual_test_losses=", actual_test_losses)
print("actual_accuracies=", actual_accuracies)
# to update expected outcomes, enable pdb and run the test with -s and copy paste outputs
# import pdb; pdb.set_trace()
rtol = 1e-03
assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch")
assert_allclose(expected_test_losses, actual_test_losses, rtol=rtol, err_msg="test loss mismatch")
assert_allclose(expected_test_accuracies, actual_accuracies, rtol=rtol, err_msg="test accuracy mismatch")
def testMNISTResumeTrainingAndTesting(self):
torch.manual_seed(1)
device = torch.device("cuda")
mnist = MNISTWrapper()
train_loader, test_loader = mnist.get_loaders()
model, model_desc = mnist.get_model()
learningRate = 0.01
args_epochs = 2
args_checkpoint_epoch = 1
# should match those in test without checkpointing
expected_losses = [0.23006735742092133, 0.48427966237068176,
0.30716797709465027, 0.3238796889781952, 0.19543828070163727, 0.3561663031578064,
0.3089643716812134, 0.37738722562789917, 0.24883587658405304, 0.30744990706443787]
expected_test_losses = [0.25183824462890625]
expected_test_accuracies = [0.9304]
actual_losses = []
actual_test_losses, actual_accuracies = [], []
# restore from checkpoint
resume_trainer = mnist.get_trainer(model, model_desc, device)
checkpoint = torch.load(get_name("ckpt_mnist.pt"), map_location="cpu")
torch.set_rng_state(checkpoint['rng_state'])
resume_trainer.load_state_dict(checkpoint['model'], strict=True)
# continue ..
for epoch in range(args_checkpoint_epoch + 1, args_epochs + 1):
actual_losses = [*actual_losses, *mnist.train_with_trainer(learningRate, resume_trainer, device, train_loader, epoch)]
test_loss, accuracy = mnist.test_with_trainer(resume_trainer, device, test_loader)
actual_test_losses = [*actual_test_losses, test_loss]
actual_accuracies = [*actual_accuracies, accuracy]
print("actual_losses=", actual_losses)
print("actual_test_losses=", actual_test_losses)
print("actual_accuracies=", actual_accuracies)
# to update expected outcomes, enable pdb and run the test with -s and copy paste outputs
# import pdb; pdb.set_trace()
rtol = 1e-03
assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch")
assert_allclose(expected_test_losses, actual_test_losses, rtol=rtol, err_msg="test loss mismatch")
assert_allclose(expected_test_accuracies, actual_accuracies, rtol=rtol, err_msg="test accuracy mismatch")
def testRunModelWithCudaCopyStream(self):
available_providers = onnxrt.get_available_providers()
if (not 'CUDAExecutionProvider' in available_providers):
print("Skipping testRunModelWithCudaCopyStream when CUDA is not available")
else:
# adapted from issue #4829 for a race condition when copy is not on default stream
# note:
# 1. if there are intermittent failure in this test, something is wrong
# 2. it's easier to repro on slower GPU (like M60, Geforce 1070)
# to repro #4829, set the CUDA EP do_copy_in_default_stream option to False
providers = [("CUDAExecutionProvider", {"do_copy_in_default_stream": True}), "CPUExecutionProvider"]
session = onnxrt.InferenceSession(get_name("issue4829.onnx"), providers=providers)
shape = np.array([2,2], dtype=np.int64)
for iteration in range(100000):
result = session.run(output_names=['output'], input_feed={'shape': shape})
def test_bind_input_and_bind_output_with_ortvalues(self):
session = onnxrt.InferenceSession(
get_name("mul_1.onnx"), providers=onnxrt.get_available_providers())
io_binding = session.io_binding()
# Bind ortvalue as input
input_ortvalue = self.create_ortvalue_input_on_gpu()
io_binding.bind_ortvalue_input("X", input_ortvalue)
# Bind ortvalue as output
output_ortvalue = self.create_uninitialized_ortvalue_input_on_gpu()
io_binding.bind_ortvalue_output("Y", output_ortvalue)
# Sync if different CUDA streams
io_binding.synchronize_inputs()
# Invoke Run
session.run_with_iobinding(io_binding)
# Sync if different CUDA streams
io_binding.synchronize_outputs()
# Inspect contents of output_ortvalue and make sure that it has the right contents
self.assertTrue(
np.array_equal(self.create_expected_output(),
output_ortvalue.numpy()))
# Bind another ortvalue as input
input_ortvalue_2 = self.create_ortvalue_alternate_input_on_gpu()
io_binding.bind_ortvalue_input("X", input_ortvalue_2)
# Sync if different CUDA streams
io_binding.synchronize_inputs()
# Invoke Run
session.run_with_iobinding(io_binding)
# Sync if different CUDA streams
io_binding.synchronize_outputs()
# Inspect contents of output_ortvalue and make sure that it has the right contents
self.assertTrue(
np.array_equal(self.create_expected_output_alternate(),
output_ortvalue.numpy()))
def setDeviceIdTest(i):
import ctypes
import onnxruntime as onnxrt
device = ctypes.c_int()
result = ctypes.c_int()
error_str = ctypes.c_char_p()
sess = onnxrt.InferenceSession(get_name("mul_1.onnx"))
option = {'device_id': i}
sess.set_providers(['CUDAExecutionProvider'], [option])
self.assertEqual(['CUDAExecutionProvider', 'CPUExecutionProvider'], sess.get_providers())
result = cuda.cuCtxGetDevice(ctypes.byref(device))
if result != CUDA_SUCCESS:
cuda.cuGetErrorString(result, ctypes.byref(error_str))
print("cuCtxGetDevice failed with error code %d: %s" % (result, error_str.value.decode()))
self.assertEqual(result, CUDA_SUCCESS)
self.assertEqual(i, device.value)
def testLabelEncoder(self):
sess = onnxrt.InferenceSession(
get_name("LabelEncoder.onnx"),
providers=onnxrt.get_available_providers())
input_name = sess.get_inputs()[0].name
self.assertEqual(input_name, "input")
input_type = str(sess.get_inputs()[0].type)
self.assertEqual(input_type, "tensor(string)")
input_shape = sess.get_inputs()[0].shape
self.assertEqual(input_shape, [1, 1])
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "variable")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, "tensor(int64)")
output_shape = sess.get_outputs()[0].shape
self.assertEqual(output_shape, [1, 1])
# Array
x = np.array([['4']])
res = sess.run([output_name], {input_name: x})
output_expected = np.array([[3]], dtype=np.int64)
np.testing.assert_allclose(output_expected,
res[0],
rtol=1e-05,
atol=1e-08)
# Python type
x = np.array(['4'], ndmin=2)
res = sess.run([output_name], {input_name: x})
output_expected = np.array([3], ndmin=2, dtype=np.int64)
np.testing.assert_allclose(output_expected,
res[0],
rtol=1e-05,
atol=1e-08)
x = np.array(['4'], ndmin=2, dtype=object)
res = sess.run([output_name], {input_name: x})
output_expected = np.array([3], ndmin=2, dtype=np.int64)
np.testing.assert_allclose(output_expected,
res[0],
rtol=1e-05,
atol=1e-08)
def testInputBytes(self):
sess = onnxrt.InferenceSession(get_name("identity_string.onnx"))
x = np.array([b'this', b'is', b'identity', b'test']).reshape((2, 2))
x_name = sess.get_inputs()[0].name
self.assertEqual(x_name, "input:0")
x_shape = sess.get_inputs()[0].shape
self.assertEqual(x_shape, [2, 2])
x_type = sess.get_inputs()[0].type
self.assertEqual(x_type, 'tensor(string)')
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "output:0")
output_shape = sess.get_outputs()[0].shape
self.assertEqual(output_shape, [2, 2])
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, 'tensor(string)')
res = sess.run([output_name], {x_name: x})
np.testing.assert_equal(x, res[0].astype('|S8'))
h1.pack_start(b2, True, True, 0)
box2.pack_start(h1, False, False, 0)
self.window.add(box2)
self.window.show_all()
def main(self):
thread.start_new_thread (recv_msg, ("Thread-1", ))
gtk.main()
if __name__ == "__main__":
name = ''
while name=='':
name = helper.get_name()
if name==False:
sys.exit()
host, port = helper.get_connection_details()
while host=='' or port=='':
host, port = helper.get_connection_details()
if host==False and port==False:
sys.exit()
sock_name = {}
sock_name['<server>'] = name + ' : '
sock_name[name] = name
CONNECTION_LIST = []
flag = 0
