def test_save_and_load_tensor(self): # type: () -> None
proto = self._simple_tensor()
cls = TensorProto
proto_string = onnx._serialize(proto)
# Test if input is string
loaded_proto = onnx.load_tensor_from_string(proto_string)
self.assertTrue(proto == loaded_proto)
# Test if input has a read function
f = io.BytesIO()
onnx.save_tensor(loaded_proto, f)
f = io.BytesIO(f.getvalue())
loaded_proto = onnx.load_tensor(f, cls)
self.assertTrue(proto == loaded_proto)
# Test if input is a file name
try:
tfile = tempfile.NamedTemporaryFile(delete=False)
onnx.save_tensor(proto, tfile)
tfile.close()
loaded_proto = onnx.load_tensor(tfile.name, cls)
self.assertTrue(proto == loaded_proto)
finally:
os.remove(tfile.name)
def main():
from_dir = sys.argv[1]
to_dir = sys.argv[2]
os.makedirs(to_dir, exist_ok=True)
xmodel = onnx.load(os.path.join(from_dir, 'model.onnx'))
convert_model(xmodel)
onnx.save(xmodel, os.path.join(to_dir, 'model.onnx'))
for test_dir in glob.glob(os.path.join(from_dir, 'test_data_set_*')):
dir_name = os.path.basename(test_dir)
to_test_dir = os.path.join(to_dir, dir_name)
os.makedirs(to_test_dir, exist_ok=True)
for pb_filename in glob.glob(os.path.join(test_dir, '*.pb')):
pb_name = os.path.basename(pb_filename)
tensor = onnx.load_tensor(pb_filename)
convert_tensor(tensor)
onnx.save_tensor(tensor, os.path.join(to_test_dir, pb_name))
assert len(inputs) == 1
input = inputs[0]
old_shape = []
new_shape = []
for i in range(len(input.type.tensor_type.shape.dim)):
nd = input.type.tensor_type.shape.dim[i].dim_value
old_shape.append(nd)
if i >= 2:
nd = int(nd * ratio)
new_shape.append(nd)
input.type.tensor_type.shape.dim[i].dim_value = nd
print('new_shape', new_shape)
onnx.save(model, os.path.join(output_dir, 'model.onnx'))
input_pb = os.path.join(output_dir, 'test_data_set_0/input_0.pb')
input_tensor = onnx.load_tensor(input_pb)
assert len(input_tensor.dims) == len(new_shape)
input = onnx.numpy_helper.to_array(input_tensor)
pad_width = []
for od, nd in zip(old_shape, new_shape):
pad_width.append((0, nd - od))
input = np.pad(input, pad_width, 'constant')
onnx.save_tensor(onnx.numpy_helper.from_array(input), input_pb)
def test_bidaf(self):
# download BiDAF model
cwd = os.getcwd()
bidaf_url = 'https://onnxzoo.blob.core.windows.net/models/opset_9/bidaf/bidaf.tar.gz'
cache_dir = os.path.join(os.path.expanduser("~"), '.cache',
'onnxruntime')
os.makedirs(cache_dir, exist_ok=True)
bidaf_local = os.path.join(cache_dir, 'bidaf.tar.gz')
if not os.path.exists(bidaf_local):
urllib.request.urlretrieve(bidaf_url, bidaf_local)
with tarfile.open(bidaf_local, 'r') as f:
f.extractall(cwd)
# verify accuracy of quantized model
bidaf_dir = os.path.join(cwd, 'bidaf')
bidaf_model = os.path.join(bidaf_dir, 'bidaf.onnx')
bidaf_scan_model = os.path.join(bidaf_dir, 'bidaf_scan.onnx')
bidaf_opt_scan_model = os.path.join(bidaf_dir, 'bidaf_opt_scan.onnx')
bidaf_int8_scan_only_model = os.path.join(bidaf_dir,
'bidaf_int8_scan_only.onnx')
subprocess.run([
sys.executable, '-m', 'onnxruntime.nuphar.model_editor', '--input',
bidaf_model, '--output', bidaf_scan_model, '--mode', 'to_scan'
],
check=True,
cwd=cwd)
subprocess.run([
sys.executable, '-m', 'onnxruntime.nuphar.model_editor', '--input',
bidaf_scan_model, '--output', bidaf_opt_scan_model, '--mode',
'opt_inproj'
],
check=True,
cwd=cwd)
subprocess.run([
sys.executable, '-m', 'onnxruntime.nuphar.model_quantizer',
'--input', bidaf_opt_scan_model, '--output',
bidaf_int8_scan_only_model, '--only_for_scan'
],
check=True,
cwd=cwd)
# run onnx_test_runner to verify results
# use -M to disable memory pattern
onnx_test_runner = os.path.join(cwd, 'onnx_test_runner')
subprocess.run([
onnx_test_runner, '-e', 'nuphar', '-M', '-c', '1', '-j', '1', '-n',
'bidaf', cwd
],
check=True,
cwd=cwd)
# test AOT on the quantized model
if os.name not in ['nt', 'posix']:
return # don't run the rest of test if AOT is not supported
cache_dir = os.path.join(cwd, 'nuphar_cache')
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
# prepare feed
feed = {}
for i in range(4):
tp = onnx.load_tensor(
os.path.join(bidaf_dir, 'test_data_set_0',
'input_{}.pb'.format(i)))
feed[tp.name] = numpy_helper.to_array(tp)
for model in [bidaf_opt_scan_model, bidaf_int8_scan_only_model]:
nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir)
for isa in ['avx', 'avx2', 'avx512']:
onnxrt.capi._pybind_state.set_nuphar_settings(
nuphar_settings + ', nuphar_codegen_target:' + isa)
sess = onnxrt.InferenceSession(
model) # JIT cache happens when initializing session
cache_dir_content = os.listdir(cache_dir)
assert len(cache_dir_content) == 1
cache_versioned_dir = os.path.join(cache_dir, cache_dir_content[0])
so_name = os.path.basename(model) + '.so'
subprocess.run([
sys.executable, '-m', 'onnxruntime.nuphar.create_shared',
'--input_dir', cache_versioned_dir, '--output_name', so_name
],
check=True)
nuphar_settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_force_no_jit:{}'.format(
cache_dir, so_name, 'on')
onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings)
sess = onnxrt.InferenceSession(model)
sess.run([], feed)
# test avx
nuphar_settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_force_no_jit:{}, nuphar_codegen_target:{}'.format(
cache_dir, so_name, 'on', 'avx')
onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings)
sess = onnxrt.InferenceSession(model)
sess.run([], feed)
assert len(inputs) == 1
input = inputs[0]
old_shape = []
new_shape = []
for i in range(len(input.type.tensor_type.shape.dim)):
nd = input.type.tensor_type.shape.dim[i].dim_value
old_shape.append(nd)
if i >= 2:
nd = int(nd * ratio)
new_shape.append(nd)
input.type.tensor_type.shape.dim[i].dim_value = nd
print('new_shape', new_shape)
onnx.save(model, os.path.join(output_dir, 'model.onnx'))
input_pb = os.path.join(output_dir, 'test_data_set_0/input_0.pb')
input_tensor = onnx.load_tensor(input_pb)
assert len(input_tensor.dims) == len(new_shape)
input = onnx.numpy_helper.to_array(input_tensor)
pad_width = []
for od, nd in zip(old_shape, new_shape):
pad_width.append((0, nd - od))
input = np.pad(input, pad_width, 'constant')
onnx.save_tensor(onnx.numpy_helper.from_array(input), input_pb)
def read_tensorproto_pb_file(filename):
"""Return tuple of tensor name and numpy.ndarray of the data from a pb file containing a TensorProto."""
tensor = onnx.load_tensor(filename)
np_array = numpy_helper.to_array(tensor)
return tensor.name, np_array
def load_tensor(path):
tensor = onnx.load_tensor(path)
return onnx.numpy_helper.to_array(tensor)
def test_bidaf(self):
# download BiDAF model
cwd = os.getcwd()
bidaf_url = 'https://onnxzoo.blob.core.windows.net/models/opset_9/bidaf/bidaf.tar.gz'
cache_dir = os.path.join(os.path.expanduser("~"), '.cache','onnxruntime')
os.makedirs(cache_dir, exist_ok=True)
bidaf_local = os.path.join(cache_dir, 'bidaf.tar.gz')
if not os.path.exists(bidaf_local):
urllib.request.urlretrieve(bidaf_url, bidaf_local)
with tarfile.open(bidaf_local, 'r') as f:
f.extractall(cwd)
# verify accuracy of quantized model
bidaf_dir = os.path.join(cwd, 'bidaf')
bidaf_model = os.path.join(bidaf_dir, 'bidaf.onnx')
bidaf_scan_model = os.path.join(bidaf_dir, 'bidaf_scan.onnx')
bidaf_int8_scan_only_model = os.path.join(bidaf_dir, 'bidaf_int8_scan_only.onnx')
subprocess.run([sys.executable, 'model_editor.py', '--input', bidaf_model, '--output', bidaf_scan_model, '--mode', 'to_scan'], check=True, cwd=cwd)
subprocess.run([sys.executable, 'model_quantizer.py', '--input', bidaf_scan_model, '--output', bidaf_int8_scan_only_model, '--only_for_scan'], check=True, cwd=cwd)
# run onnx_test_runner to verify results
# use -M to disable memory pattern
# use -j 1 -c 1 to run one model/session at a time when running multiple models
onnx_test_runner = os.path.join(cwd, 'onnx_test_runner')
subprocess.run([onnx_test_runner, '-e', 'nuphar', '-M', '-c', '1', '-j', '1', '-n', 'bidaf', cwd], check=True, cwd=cwd)
# test AOT on the quantized model
cache_dir = os.path.join(cwd, 'nuphar_cache')
if os.path.exists(cache_dir):
for sub_dir in os.listdir(cache_dir):
full_sub_dir = os.path.join(cache_dir, sub_dir)
if os.path.isdir(full_sub_dir):
for f in os.listdir(full_sub_dir):
os.remove(os.path.join(full_sub_dir, f))
else:
os.makedirs(cache_dir)
nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir)
onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings)
# prepare feed
feed = {}
for i in range(4):
tp = onnx.load_tensor(os.path.join(bidaf_dir, 'test_data_set_0', 'input_{}.pb'.format(i)))
feed[tp.name] = numpy_helper.to_array(tp)
sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session
assert 'NupharExecutionProvider' in sess.get_providers()
output = sess.run([], feed)
cache_dir_content = os.listdir(cache_dir)
assert len(cache_dir_content) == 1
cache_versioned_dir = os.path.join(cache_dir, cache_dir_content[0])
if os.name == 'nt': # Windows
subprocess.run(['cmd', '/c', os.path.join(cwd, 'create_shared.cmd'), cache_versioned_dir], check=True, cwd=cwd)
elif os.name == 'posix': #Linux
subprocess.run(['bash', os.path.join(cwd, 'create_shared.sh'), '-c', cache_versioned_dir], check=True, cwd=cwd)
else:
return # don't run the rest of test if AOT is not supported
nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) + ', nuphar_cache_force_no_jit:{}'.format('on')
onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings)
sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session
assert 'NupharExecutionProvider' in sess.get_providers()
sess.run([], feed)
def read_tensor(filename):
with open(filename, 'rb') as file:
return _get_tensor_data(onnx.load_tensor(file))
print(len(tensor.raw_data))
print(dir(tensor))
#tensor.dims[:] = [672, 672]
np_array = numpy_helper.to_array(tensor)
print("Name:", tensor.name)
print("Data Type:", tensor.data_type)
print("Shape:", np_array.shape)
print(np_array)
def numpy_to_pb(name, np_data, out_filename):
tensor = numpy_helper.from_array(np_data, name)
onnx.save_tensor(tensor, out_filename)
if __name__ == '__main__':
numpy.set_printoptions(threshold=sys.maxsize)
#print_pb_file(
# "test/super_resolution/test_data_set_0/input_0.pb")
#print_pb_file(
# "test/super_resolution/test_data_set_0/output_0.pb")
tensor = onnx.load_tensor(
"test/super_resolution/test_data_set_0/output_0.pb")
tensor.dims[:] = [1, 1, 672, 672]
onnx.save_tensor(
tensor, "test/super_resolution/test_data_set_0/output_0_fixed.pb")
#print_pb_file(
# "../test/node/test_maxpool_2d_same_upper/test_data_set_0/output_0.pb")
