def test_save_and_load_tensor(self): # type: () -> None proto = self._simple_tensor() cls = TensorProto proto_string = onnx._serialize(proto) # Test if input is string loaded_proto = onnx.load_tensor_from_string(proto_string) self.assertTrue(proto == loaded_proto) # Test if input has a read function f = io.BytesIO() onnx.save_tensor(loaded_proto, f) f = io.BytesIO(f.getvalue()) loaded_proto = onnx.load_tensor(f, cls) self.assertTrue(proto == loaded_proto) # Test if input is a file name try: tfile = tempfile.NamedTemporaryFile(delete=False) onnx.save_tensor(proto, tfile) tfile.close() loaded_proto = onnx.load_tensor(tfile.name, cls) self.assertTrue(proto == loaded_proto) finally: os.remove(tfile.name)
def main(): from_dir = sys.argv[1] to_dir = sys.argv[2] os.makedirs(to_dir, exist_ok=True) xmodel = onnx.load(os.path.join(from_dir, 'model.onnx')) convert_model(xmodel) onnx.save(xmodel, os.path.join(to_dir, 'model.onnx')) for test_dir in glob.glob(os.path.join(from_dir, 'test_data_set_*')): dir_name = os.path.basename(test_dir) to_test_dir = os.path.join(to_dir, dir_name) os.makedirs(to_test_dir, exist_ok=True) for pb_filename in glob.glob(os.path.join(test_dir, '*.pb')): pb_name = os.path.basename(pb_filename) tensor = onnx.load_tensor(pb_filename) convert_tensor(tensor) onnx.save_tensor(tensor, os.path.join(to_test_dir, pb_name))
assert len(inputs) == 1 input = inputs[0] old_shape = [] new_shape = [] for i in range(len(input.type.tensor_type.shape.dim)): nd = input.type.tensor_type.shape.dim[i].dim_value old_shape.append(nd) if i >= 2: nd = int(nd * ratio) new_shape.append(nd) input.type.tensor_type.shape.dim[i].dim_value = nd print('new_shape', new_shape) onnx.save(model, os.path.join(output_dir, 'model.onnx')) input_pb = os.path.join(output_dir, 'test_data_set_0/input_0.pb') input_tensor = onnx.load_tensor(input_pb) assert len(input_tensor.dims) == len(new_shape) input = onnx.numpy_helper.to_array(input_tensor) pad_width = [] for od, nd in zip(old_shape, new_shape): pad_width.append((0, nd - od)) input = np.pad(input, pad_width, 'constant') onnx.save_tensor(onnx.numpy_helper.from_array(input), input_pb)
def test_bidaf(self): # download BiDAF model cwd = os.getcwd() bidaf_url = 'https://onnxzoo.blob.core.windows.net/models/opset_9/bidaf/bidaf.tar.gz' cache_dir = os.path.join(os.path.expanduser("~"), '.cache', 'onnxruntime') os.makedirs(cache_dir, exist_ok=True) bidaf_local = os.path.join(cache_dir, 'bidaf.tar.gz') if not os.path.exists(bidaf_local): urllib.request.urlretrieve(bidaf_url, bidaf_local) with tarfile.open(bidaf_local, 'r') as f: f.extractall(cwd) # verify accuracy of quantized model bidaf_dir = os.path.join(cwd, 'bidaf') bidaf_model = os.path.join(bidaf_dir, 'bidaf.onnx') bidaf_scan_model = os.path.join(bidaf_dir, 'bidaf_scan.onnx') bidaf_opt_scan_model = os.path.join(bidaf_dir, 'bidaf_opt_scan.onnx') bidaf_int8_scan_only_model = os.path.join(bidaf_dir, 'bidaf_int8_scan_only.onnx') subprocess.run([ sys.executable, '-m', 'onnxruntime.nuphar.model_editor', '--input', bidaf_model, '--output', bidaf_scan_model, '--mode', 'to_scan' ], check=True, cwd=cwd) subprocess.run([ sys.executable, '-m', 'onnxruntime.nuphar.model_editor', '--input', bidaf_scan_model, '--output', bidaf_opt_scan_model, '--mode', 'opt_inproj' ], check=True, cwd=cwd) subprocess.run([ sys.executable, '-m', 'onnxruntime.nuphar.model_quantizer', '--input', bidaf_opt_scan_model, '--output', bidaf_int8_scan_only_model, '--only_for_scan' ], check=True, cwd=cwd) # run onnx_test_runner to verify results # use -M to disable memory pattern onnx_test_runner = os.path.join(cwd, 'onnx_test_runner') subprocess.run([ onnx_test_runner, '-e', 'nuphar', '-M', '-c', '1', '-j', '1', '-n', 'bidaf', cwd ], check=True, cwd=cwd) # test AOT on the quantized model if os.name not in ['nt', 'posix']: return # don't run the rest of test if AOT is not supported cache_dir = os.path.join(cwd, 'nuphar_cache') if os.path.exists(cache_dir): shutil.rmtree(cache_dir) os.makedirs(cache_dir) # prepare feed feed = {} for i in range(4): tp = onnx.load_tensor( os.path.join(bidaf_dir, 'test_data_set_0', 'input_{}.pb'.format(i))) feed[tp.name] = numpy_helper.to_array(tp) for model in [bidaf_opt_scan_model, bidaf_int8_scan_only_model]: nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) for isa in ['avx', 'avx2', 'avx512']: onnxrt.capi._pybind_state.set_nuphar_settings( nuphar_settings + ', nuphar_codegen_target:' + isa) sess = onnxrt.InferenceSession( model) # JIT cache happens when initializing session cache_dir_content = os.listdir(cache_dir) assert len(cache_dir_content) == 1 cache_versioned_dir = os.path.join(cache_dir, cache_dir_content[0]) so_name = os.path.basename(model) + '.so' subprocess.run([ sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', cache_versioned_dir, '--output_name', so_name ], check=True) nuphar_settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_force_no_jit:{}'.format( cache_dir, so_name, 'on') onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) sess = onnxrt.InferenceSession(model) sess.run([], feed) # test avx nuphar_settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_force_no_jit:{}, nuphar_codegen_target:{}'.format( cache_dir, so_name, 'on', 'avx') onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) sess = onnxrt.InferenceSession(model) sess.run([], feed)
def read_tensorproto_pb_file(filename): """Return tuple of tensor name and numpy.ndarray of the data from a pb file containing a TensorProto.""" tensor = onnx.load_tensor(filename) np_array = numpy_helper.to_array(tensor) return tensor.name, np_array
def load_tensor(path): tensor = onnx.load_tensor(path) return onnx.numpy_helper.to_array(tensor)
def test_bidaf(self): # download BiDAF model cwd = os.getcwd() bidaf_url = 'https://onnxzoo.blob.core.windows.net/models/opset_9/bidaf/bidaf.tar.gz' cache_dir = os.path.join(os.path.expanduser("~"), '.cache','onnxruntime') os.makedirs(cache_dir, exist_ok=True) bidaf_local = os.path.join(cache_dir, 'bidaf.tar.gz') if not os.path.exists(bidaf_local): urllib.request.urlretrieve(bidaf_url, bidaf_local) with tarfile.open(bidaf_local, 'r') as f: f.extractall(cwd) # verify accuracy of quantized model bidaf_dir = os.path.join(cwd, 'bidaf') bidaf_model = os.path.join(bidaf_dir, 'bidaf.onnx') bidaf_scan_model = os.path.join(bidaf_dir, 'bidaf_scan.onnx') bidaf_int8_scan_only_model = os.path.join(bidaf_dir, 'bidaf_int8_scan_only.onnx') subprocess.run([sys.executable, 'model_editor.py', '--input', bidaf_model, '--output', bidaf_scan_model, '--mode', 'to_scan'], check=True, cwd=cwd) subprocess.run([sys.executable, 'model_quantizer.py', '--input', bidaf_scan_model, '--output', bidaf_int8_scan_only_model, '--only_for_scan'], check=True, cwd=cwd) # run onnx_test_runner to verify results # use -M to disable memory pattern # use -j 1 -c 1 to run one model/session at a time when running multiple models onnx_test_runner = os.path.join(cwd, 'onnx_test_runner') subprocess.run([onnx_test_runner, '-e', 'nuphar', '-M', '-c', '1', '-j', '1', '-n', 'bidaf', cwd], check=True, cwd=cwd) # test AOT on the quantized model cache_dir = os.path.join(cwd, 'nuphar_cache') if os.path.exists(cache_dir): for sub_dir in os.listdir(cache_dir): full_sub_dir = os.path.join(cache_dir, sub_dir) if os.path.isdir(full_sub_dir): for f in os.listdir(full_sub_dir): os.remove(os.path.join(full_sub_dir, f)) else: os.makedirs(cache_dir) nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) # prepare feed feed = {} for i in range(4): tp = onnx.load_tensor(os.path.join(bidaf_dir, 'test_data_set_0', 'input_{}.pb'.format(i))) feed[tp.name] = numpy_helper.to_array(tp) sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session assert 'NupharExecutionProvider' in sess.get_providers() output = sess.run([], feed) cache_dir_content = os.listdir(cache_dir) assert len(cache_dir_content) == 1 cache_versioned_dir = os.path.join(cache_dir, cache_dir_content[0]) if os.name == 'nt': # Windows subprocess.run(['cmd', '/c', os.path.join(cwd, 'create_shared.cmd'), cache_versioned_dir], check=True, cwd=cwd) elif os.name == 'posix': #Linux subprocess.run(['bash', os.path.join(cwd, 'create_shared.sh'), '-c', cache_versioned_dir], check=True, cwd=cwd) else: return # don't run the rest of test if AOT is not supported nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) + ', nuphar_cache_force_no_jit:{}'.format('on') onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session assert 'NupharExecutionProvider' in sess.get_providers() sess.run([], feed)
def read_tensor(filename): with open(filename, 'rb') as file: return _get_tensor_data(onnx.load_tensor(file))
print(len(tensor.raw_data)) print(dir(tensor)) #tensor.dims[:] = [672, 672] np_array = numpy_helper.to_array(tensor) print("Name:", tensor.name) print("Data Type:", tensor.data_type) print("Shape:", np_array.shape) print(np_array) def numpy_to_pb(name, np_data, out_filename): tensor = numpy_helper.from_array(np_data, name) onnx.save_tensor(tensor, out_filename) if __name__ == '__main__': numpy.set_printoptions(threshold=sys.maxsize) #print_pb_file( # "test/super_resolution/test_data_set_0/input_0.pb") #print_pb_file( # "test/super_resolution/test_data_set_0/output_0.pb") tensor = onnx.load_tensor( "test/super_resolution/test_data_set_0/output_0.pb") tensor.dims[:] = [1, 1, 672, 672] onnx.save_tensor( tensor, "test/super_resolution/test_data_set_0/output_0_fixed.pb") #print_pb_file( # "../test/node/test_maxpool_2d_same_upper/test_data_set_0/output_0.pb")