def save_onnx_model(self, model_proto, feed_dict):
save_path = os.path.join(type(self).TMPPATH, self._testMethodName)
target_path = utils.save_onnx_model(save_path, self._testMethodName, feed_dict,
model_proto, include_test_data=self.debug_mode())
self.log.debug("create model file: %s", target_path)
return target_path
def run_onnxruntime(self,
name,
model_proto,
inputs,
external_tensor_storage=None):
"""Run test against onnxruntime backend."""
import onnxruntime as rt
model_path = utils.save_onnx_model(
TEMP_DIR,
name,
inputs,
model_proto,
include_test_data=True,
as_text=utils.is_debug_mode(),
external_tensor_storage=external_tensor_storage)
logger.info("Model saved to %s", model_path)
if self.use_custom_ops:
from ortcustomops import get_library_path
opt = rt.SessionOptions()
opt.register_custom_ops_library(get_library_path())
m = rt.InferenceSession(model_path, opt)
else:
m = rt.InferenceSession(model_path)
results = m.run(self.output_names, inputs)
if self.perf:
start = time.time()
for _ in range(PERFITER):
_ = m.run(self.output_names, inputs)
self.onnx_runtime = time.time() - start
return results
def save_onnx_model(self, model_proto, feed_dict, postfix=""):
target_path = utils.save_onnx_model(self.test_data_directory, self._testMethodName + postfix, feed_dict,
model_proto, include_test_data=self.config.is_debug_mode,
as_text=self.config.is_debug_mode)
self.logger.debug("create model file: %s", target_path)
return target_path
def run_onnxruntime(self, name, model_proto, inputs):
"""Run test against msrt-next backend."""
import onnxruntime as rt
model_path = utils.save_onnx_model(TMPPATH, name, inputs, model_proto)
m = rt.InferenceSession(model_path)
results = m.run(self.output_names, inputs)
if self.perf:
start = time.time()
for _ in range(PERFITER):
_ = m.run(self.output_names, inputs)
self.onnx_runtime = time.time() - start
return results
def run_onnxruntime(self, name, model_proto, inputs):
"""Run test against msrt-next backend."""
import onnxruntime as rt
model_path = utils.save_onnx_model(TEMP_DIR, name, inputs, model_proto, include_test_data=True)
logger.info("Model saved to %s", model_path)
m = rt.InferenceSession(model_path)
results = m.run(self.output_names, inputs)
if self.perf:
start = time.time()
for _ in range(PERFITER):
_ = m.run(self.output_names, inputs)
self.onnx_runtime = time.time() - start
return results
def run_onnxruntime(self,
name,
model_proto,
inputs,
outputs,
external_tensor_storage=None):
"""Run test against onnxruntime backend."""
import onnxruntime as rt
model_path = utils.save_onnx_model(
TEMP_DIR,
name,
inputs,
model_proto,
include_test_data=True,
as_text=utils.is_debug_mode(),
external_tensor_storage=external_tensor_storage)
logger.info("Model saved to %s", model_path)
providers = ['CPUExecutionProvider']
if rt.get_device() == "GPU":
gpus = os.environ.get("CUDA_VISIBLE_DEVICES")
if gpus is None or len(gpus) > 1:
providers = ['CUDAExecutionProvider']
opt = rt.SessionOptions()
if self.use_custom_ops:
from onnxruntime_extensions import get_library_path
opt.register_custom_ops_library(get_library_path())
if self.ort_profile is not None:
opt.enable_profiling = True
m = rt.InferenceSession(model_path,
sess_options=opt,
providers=providers)
results = m.run(outputs, inputs)
if self.perf:
n = 0
start = time.time()
stop = start + PERF_TIME
while time.time() < stop:
for _ in range(PERF_STEP):
_ = m.run(outputs, inputs)
n += PERF_STEP
self.onnx_runtime = 1000 * (time.time() - start) / n
logger.info("ORT perf {:.2f}ms/inference, n={}".format(
self.onnx_runtime, n))
if self.ort_profile is not None:
tmp_path = m.end_profiling()
shutil.move(tmp_path, self.ort_profile)
return results
def run_onnxruntime(self,
name,
model_proto,
inputs,
outputs,
external_tensor_storage=None):
"""Run test against onnxruntime backend."""
import onnxruntime as rt
model_path = utils.save_onnx_model(
TEMP_DIR,
name,
inputs,
model_proto,
include_test_data=True,
as_text=utils.is_debug_mode(),
external_tensor_storage=external_tensor_storage)
logger.info("Model saved to %s", model_path)
opt = rt.SessionOptions()
if self.use_custom_ops:
from ortcustomops import get_library_path
opt.register_custom_ops_library(get_library_path())
m = rt.InferenceSession(model_path, opt)
if self.ort_profile is not None:
opt.enable_profiling = True
m = rt.InferenceSession(model_path, opt)
results = m.run(outputs, inputs)
if self.perf:
n = 0
start = time.time()
stop = start + PERF_TIME
while time.time() < stop:
for _ in range(PERF_STEP):
_ = m.run(outputs, inputs)
n += PERF_STEP
self.onnx_runtime = 1000 * (time.time() - start) / n
logger.info("ORT perf {:.2f}ms/inference, n={}".format(
self.onnx_runtime, n))
if self.ort_profile is not None:
tmp_path = m.end_profiling()
shutil.move(tmp_path, self.ort_profile)
return results
# [array([[[-0.05846359 -0.06566401 0.02254938 -0.26033643 -0.07923548]],
# [[ 0.04879569 0.04215769 -0.06720451 -0.60583305 0.06223793]],
# [[-0.05626901 -0.06627436 0.00422506 -0.5533649 -0.0767431 ]]], dtype=float32)]
with tf.Session() as sess:
# output_dict: get tensor by output name
output_dict = []
for out_name in output_names_with_port:
output_dict.append(sess.graph.get_tensor_by_name(out_name))
expected = sess.run(output_dict, feed_dict={"input_1:0": x_val})
# tf optimize
graph_def = tf_optimize(input_names_with_port,
output_names_with_port,
sess.graph_def,
fold_constant=True)
tf.reset_default_graph()
tf.import_graph_def(graph_def, name='')
# convert to onnx
with tf.Session() as sess:
g = process_tf_graph(sess.graph, output_names=output_names_with_port)
g = optimizer.optimize_graph(g)
model_proto = g.make_model("lstm")
utils.save_onnx_model("./models",
"lstm",
feed_dict={"input_1:0": input},
model_proto=model_proto)