def _make_value_info(self, variable):
value_info = helper.ValueInfoProto()
value_info.name = variable.full_name
value_info.type.CopyFrom(variable.type.to_onnx_type())
if variable.type.doc_string:
value_info.doc_string = variable.type.doc_string
return value_info
def add_fmaps_to_outputs(self):
model_nodes = self.onnx_model.graph.node
model_outputs = [n.name for n in self.onnx_model.graph.output]
for node in model_nodes:
if node.output[0] in model_outputs:
continue
logging.info("{} -> {} ({})".format(node.name, node.output, len(node.output)))
intermediate_layer_value_info = helper.ValueInfoProto()
intermediate_layer_value_info.name = node.output[0]
self.onnx_model.graph.output.append(intermediate_layer_value_info)
onnx.save(self.onnx_model, self.model_path_extended)
def get_onnx_output(model, input, intermediate_node=None):
sess = onnxruntime.InferenceSession(file_path)
x = input
x = x.astype(np.float32)
input_name = model.graph.input[0].name
if (intermediate_node != None):
intermediate_layer_value_info = helper.ValueInfoProto()
intermediate_layer_value_info.name = sys.argv[2]
model.graph.output.extend([intermediate_layer_value_info])
onnx.save(model, file_path + '_1')
sess = onnxruntime.InferenceSession(file_path + '_1')
pred = sess.run([intermediate_layer_value_info.name], {input_name: x})
return pred
pred = sess.run(None, {input_name: x})
return pred
def main():
# First read the ONNX file
if (len(sys.argv) < 2):
print("TF python file unspecified.", file=sys.stderr)
exit(1)
file_name = sys.argv[1]
file_path = 'models/' + file_name
model_name = file_name[:-5] # name without the '.onnx' extension
model = onnx.load(file_path)
model = preprocess_for_tf(model)
x = np.load('debug/' + model_name + '/' + model_name + '_input.npy')
x = x.astype(np.float32)
input_name = model.graph.input[0].name
output_name = model.graph.output[0].name
if (len(sys.argv) > 2):
intermediate_layer_value_info = helper.ValueInfoProto()
intermediate_layer_value_info_name = 'tf_' + sys.argv[2]
intermediate_layer_value_info = helper.make_tensor_value_info(
intermediate_layer_value_info_name, TensorProto.FLOAT, [])
model.graph.output.extend([intermediate_layer_value_info])
output = prepare(model).run(x)
pred = getattr(output, intermediate_layer_value_info_name)
np.save('debug/' + model_name + '/' + model_name + '_debug', pred)
with open('debug/onnx_debug.txt', 'w') as f:
f.write(common.numpy_float_array_to_float_val_str(pred))
print("Saving the onnx runtime intermediate output for " +
intermediate_layer_value_info.name)
exit()
output = prepare(model).run(x)
pred = getattr(output, output_name)
np.save('debug/' + model_name + '/' + model_name + '_output', pred)
with open('debug/onnx_output.txt', 'w') as f:
f.write(common.numpy_float_array_to_float_val_str(pred))
output_dims = common.proto_val_to_dimension_tuple(model.graph.output[0])
print("Saving the onnx runtime output of dimension " + str(output_dims))
print("TF python file unspecified.", file=sys.stderr)
exit(1)
file_name = sys.argv[1]
file_path = 'models/' + file_name
model_name = file_name[:-5] # name without the '.onnx' extension
model = onnx.load(file_path)
sess = onnxruntime.InferenceSession(file_path)
x = np.load('debug/' + model_name + '/' + model_name + '_input.npy')
x = x.astype(np.float32)
input_name = model.graph.input[0].name
if (len(sys.argv) > 2):
intermediate_layer_value_info = helper.ValueInfoProto()
intermediate_layer_value_info.name = sys.argv[2]
model.graph.output.extend([intermediate_layer_value_info])
onnx.save(model, file_path + '_1')
sess = onnxruntime.InferenceSession(file_path + '_1')
pred = sess.run([intermediate_layer_value_info.name], {input_name: x})
np.save('debug/' + model_name + '/' + model_name + '_debug', pred)
with open('debug/onnx_debug.txt', 'w') as f:
f.write(common.numpy_float_array_to_float_val_str(pred))
print("Saving the onnx runtime intermediate output for " + intermediate_layer_value_info.name)
exit()
pred = sess.run(None, {input_name: x})
np.save('debug/' + model_name + '/' + model_name + '_output', pred)
with open('debug/onnx_output.txt', 'w') as f:
f.write(common.numpy_float_array_to_float_val_str(pred))
def select_model_inputs_outputs(model, outputs=None, inputs=None):
"""
Takes a model and changes its outputs.
@param model :epkg:`ONNX` model
@param inputs new inputs, same ones if None
@param outputs new outputs, same ones if None
@return modified model
The function removes unneeded files.
"""
if inputs is not None:
raise NotImplementedError("Parameter inputs cannot be empty.")
if outputs is None:
raise RuntimeError("Parameter outputs cannot be None.")
if not isinstance(outputs, list):
outputs = [outputs]
mark_var = {}
for out in enumerate_model_node_outputs(model):
mark_var[out] = 0
for inp in model.graph.input:
mark_var[inp.name] = 0
for out in outputs:
if out not in mark_var:
raise ValueError("Output '{}' not found in model.".format(out))
mark_var[out] = 1
nodes = model.graph.node[::-1]
mark_op = {}
for node in nodes:
mark_op[node.name] = 0
# We mark all the nodes we need to keep.
nb = 1
while nb > 0:
nb = 0
for node in nodes:
if mark_op[node.name] == 1:
continue
mod = False
for out in node.output:
if mark_var[out] == 1:
mark_op[node.name] = 1
mod = True
break
if not mod:
continue
nb += 1
for inp in node.input:
if mark_var.get(inp, 0) == 1:
continue
mark_var[inp] = 1
nb += 1
# All nodes verifies mark_op[node.name] == 1
keep_nodes = [node for node in nodes if mark_op[node.name] == 1]
var_out = []
for out in outputs:
value_info = helper.ValueInfoProto()
value_info.name = out
var_out.append(value_info)
graph = helper.make_graph(keep_nodes, model.graph.name, model.graph.input,
var_out, model.graph.initializer)
onnx_model = helper.make_model(graph)
onnx_model.ir_version = model.ir_version
onnx_model.producer_name = model.producer_name
onnx_model.producer_version = model.producer_version
onnx_model.domain = model.domain
onnx_model.model_version = model.model_version
onnx_model.doc_string = model.doc_string
if len(onnx_model.graph.input) != len(model.graph.input): # pylint: disable=E1101
raise RuntimeError("Input mismatch {} != {}".format(
len(onnx_model.input), len(model.input))) # pylint: disable=E1101
return onnx_model
def augment_graph(self, activation_only=False, output_only=False):
'''
Adds nodes to all quantization_candidates op type nodes in
model and ensures their outputs are stored as part of the graph output
:param activation_only(bool): whether to dump activation tensor only
:param output_only(bool): whether to dump output_only
:return: augmented ONNX model
'''
model = copy.deepcopy(self.model)
model_nodes_names = [node.name for node in model.graph.node]
added_nodes = []
added_outputs = []
tensors_to_dump = set()
for augment_node_type in self.augment_nodes:
if augment_node_type not in [
'ReduceMin', 'ReduceMax', 'DequantizeLinear'
]:
raise ValueError("Unexpected augment_node {} only \
ReduceMin/ReduceMax are supported".format(
augment_node_type))
if self.already_quantized:
# mapping between fp32 node and int8 node
new_white_nodes = []
for white_node in self.white_nodes:
new_white_node = white_node + "_quant"
assert new_white_node in model_nodes_names, "no quantized {} \
in the graph".format(
white_node)
new_white_nodes.append(new_white_node)
self.white_nodes = new_white_nodes
initializer_names = [i.name for i in model.graph.initializer]
for node in model.graph.node: # pylint: disable=no-member
should_be_dump = ((node.op_type in self.dump_op_types) and
(node.name not in self.black_nodes)) or \
(node.name in self.white_nodes)
if should_be_dump:
if not output_only:
if node.op_type == "Attention":
if len(node.input) >= 3:
logger.debug(
"indice input {} of attention node {} is integer"
.format(node.input[3:], node.name))
tensors_to_dump.update(node.input[:2])
else:
tensors_to_dump.update(node.input)
elif node.op_type == "Gather":
logger.debug(
"indice input {} of gather node {} is integer".
format(node.input[-1], node.name))
tensors_to_dump.update(node.input[:-1])
else:
tensors_to_dump.update(node.input)
else:
for input in node.input:
if input in initializer_names:
tensors_to_dump.add(input)
tensors_to_dump.update(node.output)
tensors_tmp = set()
if activation_only:
for tensor in tensors_to_dump:
if tensor not in initializer_names: # pylint: disable=no-member
tensors_tmp.add(tensor)
tensors_to_dump = tensors_tmp
for tensor in tensors_to_dump:
if self.augment_nodes:
for augment_node_type in self.augment_nodes:
if augment_node_type in ['ReduceMin', 'ReduceMax']:
# dump tensor for calibration
augment_node_name = tensor + "_" + augment_node_type
augment_node = onnx.helper.make_node(
augment_node_type, [tensor], [augment_node_name],
augment_node_name,
keepdims=0)
added_nodes.append(augment_node)
added_outputs.append(
helper.make_tensor_value_info(
augment_node.output[0], # pylint: disable=no-member
TensorProto.FLOAT,
())) # pylint: disable=no-member
else:
# insert DequantizeLinear node as output
augment_node_name = tensor + "_new_" + augment_node_type
scale, zero_point = self.model_wrapper.get_scale_zo(
tensor)
if scale:
# the tensor is in INT8 dtype
nodes, output = self._dequantize(
tensor, scale, zero_point)
added_nodes.extend(nodes)
added_outputs.append(
helper.make_tensor_value_info(
output, # pylint: disable=no-member
TensorProto.FLOAT,
())) # pylint: disable=no-member
else:
# the tensor is in FP32 dtype
if tensor not in [
t.name for t in model.graph.output
]:
added_tensor = helper.ValueInfoProto()
added_tensor.name = tensor
added_outputs.append(added_tensor)
else:
if tensor not in [t.name for t in model.graph.output]:
added_tensor = helper.ValueInfoProto()
added_tensor.name = tensor
added_outputs.append(added_tensor)
if self.augment_nodes:
model.graph.node.extend(added_nodes) # pylint: disable=no-member
model.graph.output.extend(added_outputs) # pylint: disable=no-member
self.augmented_model = model
onnx.save(model, self.augmented_model_path)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument(
"--input_file",
help="Input npz or jpg file."
)
parser.add_argument(
"--output_file",
help="Output npz"
)
parser.add_argument(
"--model_path",
help="onnx model path."
)
parser.add_argument(
"--dump_tensors",
help="Dump all output tensors into a file in npz format"
)
parser.add_argument(
"--mean", default='0,0,0',
help="Per Channel image mean values"
)
parser.add_argument(
"--mean_file", default='mean',
help="Data set image mean of [Channels x Height x Width] dimensions " +
"(numpy array). Set to '' for no mean subtraction."
)
parser.add_argument(
"--std",
help="Per Channel image std values",
default='1,1,1'
)
parser.add_argument(
"--input_scale",
type=float,
default=1.0,
help="Multiply input features by this scale to finish preprocessing."
)
parser.add_argument(
"--raw_scale",
type=float,
default=255.0,
help="Multiply raw input by this scale before preprocessing."
)
parser.add_argument(
"--net_input_dims",
default='224,224',
help="'height,width' dimensions of network input spatial dimensions."
)
parser.add_argument(
"--image_resize_dims",
default='256,256',
help="To resize to this size first, then crop to net_input_dims."
)
args = parser.parse_args()
# preprocessor = cvi_preprocess()
# preprocessor.config(net_input_dims=args.net_input_dims,
# resize_dims=args.image_resize_dims,
# mean=args.mean,
# mean_file=args.mean_file,
# input_scale=args.input_scale,
# raw_scale=args.raw_scale,
# std=args.std)
input=None
file_extension = args.input_file.split(".")[-1].lower()
if file_extension == "jpg":
mean = [float(x) for x in args.mean.split(",")]
if args.std:
std = [float(x) for x in args.std.split(",")]
else:
std = [1, 1, 1]
net_input_dims = [int(x) for x in args.net_input_dims.split(",")]
if args.image_resize_dims:
image_resize_dims = [int(x) for x in args.image_resize_dims.split(",")]
else:
image_resize_dims = net_input_dims
input = preprocessor.run(args.input_file)
elif file_extension == "npz":
input = np.load(args.input_file)['input']
else:
print("Not support {} extension")
ort_outs = inference(input, args.model_path)
np.savez(args.output_file, **{'output': ort_outs[0]})
#print("org ort_outs", ort_outs)
if args.dump_tensors:
# second pass for dump all output
# plz refre https://github.com/microsoft/onnxruntime/issues/1455
output_keys = ['output']
model = onnx.load(args.model_path)
# tested commited #c3cea486d https://github.com/microsoft/onnxruntime.git
for x in model.graph.node:
_intermediate_tensor_name = list(x.output)
intermediate_tensor_name = ",".join(_intermediate_tensor_name)
intermediate_layer_value_info = helper.ValueInfoProto()
intermediate_layer_value_info.name = intermediate_tensor_name
model.graph.output.append(intermediate_layer_value_info)
output_keys.append(intermediate_layer_value_info.name + '_' + x.op_type)
model_file = args.model_path.split("/")[-1]
dump_all_onnx = "all_{}".format(model_file)
if not os.path.exists(dump_all_onnx):
onnx.save(model, dump_all_onnx)
else:
print("{} is exitsed!".format(dump_all_onnx))
print("dump multi-output onnx all tensor at ", dump_all_onnx)
# dump all inferneced tensor
ort_outs = inference(input, dump_all_onnx)
tensor_all_dict = dict(zip(output_keys, map(np.ndarray.flatten, ort_outs)))
tensor_all_dict['input'] = input
np.savez(args.dump_tensors, **tensor_all_dict)
print("dump all tensor at ", args.dump_tensors)
def augment_graph(self):
'''
Adds nodes to all quantization_candidates op type nodes in
model and ensures their outputs are stored as part of the graph output
:return: augmented ONNX model
'''
model = copy.deepcopy(self.model)
added_nodes = []
added_outputs = []
tensors_to_dump = set()
for node in model.graph.node: # pylint: disable=no-member
should_be_dump = ((node.op_type in self.dump_op_types) and
(node.name not in self.black_nodes)) or \
(node.name in self.white_nodes)
if should_be_dump:
if node.op_type == "Attention":
if len(node.input) >= 3:
logger.debug("indice input {} of attention node {} is in integer format"
.format(node.input[3:], node.name))
tensors_to_dump.update(node.input[:2])
else:
tensors_to_dump.update(node.input)
elif node.op_type == "Gather":
logger.debug("indice input {} of gather node {} is in integer format"
.format(node.input[-1], node.name))
tensors_to_dump.update(node.input[:-1])
else:
tensors_to_dump.update(node.input)
tensors_to_dump.update(node.output)
for tensor in tensors_to_dump:
if tensor in model.graph.initializer: # pylint: disable=no-member
tensors_to_dump.remove(tensor)
for tensor in tensors_to_dump:
if self.augment_nodes:
for augment_node_type in self.augment_nodes:
if augment_node_type not in ['ReduceMin', 'ReduceMax']:
raise ValueError("Unexpected augment_node {} only \
ReduceMin/ReduceMax are supported".format(augment_node_type))
augment_node_name = tensor + "_" + augment_node_type
augment_node = onnx.helper.make_node(augment_node_type, [tensor],
[augment_node_name],
augment_node_name,
keepdims=0)
added_nodes.append(augment_node)
added_outputs.append(helper.make_tensor_value_info(
augment_node.output[0], # pylint: disable=no-member
TensorProto.FLOAT, ())) # pylint: disable=no-member
else:
if tensor not in [t.name for t in model.graph.output]:
added_tensor = helper.ValueInfoProto()
added_tensor.name = tensor
added_outputs.append(added_tensor)
if self.augment_nodes:
model.graph.node.extend(added_nodes) # pylint: disable=no-member
model.graph.output.extend(added_outputs) # pylint: disable=no-member
self.augmented_model = model
onnx.save(model, self.augmented_model_path)
traceback.format_exc()
]) + '\n\n\n')
tested_file.flush()
continue
if dataset == 'cifar10':
input = np.array(test_input,
dtype=np.float32).reshape([1, 32, 32, 3])
elif dataset == 'mnist':
input = np.array(test_input,
dtype=np.float32).reshape([1, 28, 28, 1])
if is_onnx:
input = input.transpose(0, 3, 1, 2)
for name, shape in output_info:
out_node = helper.ValueInfoProto(type=helper.TypeProto())
out_node.name = name
out_node.type.tensor_type.elem_type = model.graph.output[
0].type.tensor_type.elem_type
if len(shape) == 4:
shape = [shape[0], shape[3], shape[1], shape[2]]
for dim_value in shape:
dim = out_node.type.tensor_type.shape.dim.add()
dim.dim_value = dim_value
model.graph.output.append(out_node)
runnable = rt.prepare(model, 'CPU')
pred = runnable.run(input)
#print(pred)
else:
if not (is_saved_tf_model or is_pb_file):
input = np.array(test_input, dtype=np.float32)
def select_model_inputs_outputs(model, outputs=None, inputs=None,
infer_shapes=False, overwrite=None,
remove_unused=True,
verbose=0, fLOG=None):
"""
Takes a model and changes its outputs.
:param model: :epkg:`ONNX` model
:param inputs: new inputs, same ones if None
:param outputs: new outputs, same ones if None
:param infer_shapes: infer inputs and outputs shapes
:param overwrite: overwrite type and shapes for
inputs or outputs, *overwrite* is a
dictionary `{'name': (numpy dtype, shape)}`
:param remove_unused: remove unused nodes from the graph
:param verbose: display information while converting
:param fLOG: logging function
:return: modified model
The function removes unneeded nodes.
.. exref::
:title: Change ONNX model inputs
The following exampels shows how to change the inputs of model
to bypass the first nodes. Shape inferences fails to determine
the new inputs type. They need to be overwritten.
`verbose=1, fLOG=print` shows the number of deleted nodes.
::
import onnx
from mlprodict.onnx_tools.onnx_manipulations import select_model_inputs_outputs
onx = onnx.load(path)
onx2 = select_model_inputs_outputs(
onx, inputs=["SentenceTokenizer/SentencepieceTokenizeOp:0",
"SentenceTokenizer/SentencepieceTokenizeOp:1"],
infer_shapes=True, verbose=1, fLOG=print,
overwrite={'SentenceTokenizer/SentencepieceTokenizeOp:0': (numpy.int32, None),
'SentenceTokenizer/SentencepieceTokenizeOp:1': (numpy.int64, None)})
onnx.save(onx2, path2)
.. versionchanged:: 0.6
Supports the case where inputs are changed.
.. versionchanged:: 0.7
Parameter *remove_unused* was added. Unused are removed by default.
"""
if inputs is not None and not isinstance(inputs, list):
inputs = [inputs]
if outputs is not None and not isinstance(outputs, list):
outputs = [outputs]
if inputs is None:
inputs = [i.name for i in model.graph.input]
if outputs is None:
outputs = [o.name for o in model.graph.output]
mark_var = {}
for out in enumerate_model_node_outputs(model):
mark_var[out] = 0
for inp in inputs:
mark_var[inp] = 0
for out in outputs:
if out not in mark_var:
raise ValueError( # pragma: no cover
"Output '{}' not found in model.".format(out))
mark_var[out] = 1
nodes = model.graph.node[::-1]
mark_op = {}
for node in nodes:
mark_op[node.name] = 0
# We mark all the nodes we need to keep.
nb = 1
while nb > 0:
nb = 0
for node in nodes:
if mark_op[node.name] == 1:
continue
mod = False
for out in node.output:
if mark_var[out] == 1:
mark_op[node.name] = 1
mod = True
break
if not mod:
continue
nb += 1
for inp in node.input:
if inp in inputs:
continue
if mark_var.get(inp, 0) == 1:
continue
mark_var[inp] = 1
nb += 1
# All nodes verifies mark_op[node.name] == 1
keep_nodes = [node for node in nodes if mark_op[node.name] == 1]
known_shapes = {}
if infer_shapes:
shapes = shape_inference.infer_shapes(model)
for shape in shapes.graph.value_info: # pylint: disable=E1101
known_shapes[shape.name] = shape.type
for shape in shapes.graph.input: # pylint: disable=E1101
known_shapes[shape.name] = shape.type
for shape in shapes.graph.output: # pylint: disable=E1101
known_shapes[shape.name] = shape.type
else:
for shape in model.graph.input: # pylint: disable=E1101
known_shapes[shape.name] = shape.type
for shape in model.graph.output: # pylint: disable=E1101
known_shapes[shape.name] = shape.type
var_in = []
for name in inputs:
if overwrite is not None and name in overwrite:
dtype, shape = overwrite[name]
proto_dtype = guess_proto_dtype(dtype)
value_info = helper.make_tensor_value_info(
name, proto_dtype, shape)
elif name in known_shapes:
info = known_shapes[name].tensor_type
proto_dtype = info.elem_type
if proto_dtype == 0:
value_info = helper.ValueInfoProto()
value_info.name = name
else:
shape = [getattr(d, 'dim_value', None) for d in info.shape.dim]
if len(shape) == 0:
shape = None
else:
shape = [None if s == 0 else s for s in shape]
value_info = helper.make_tensor_value_info(
name, proto_dtype, shape)
else:
value_info = helper.ValueInfoProto()
value_info.name = name
var_in.append(value_info)
var_out = []
for name in outputs:
if overwrite is not None and name in overwrite:
dtype, shape = overwrite[name]
proto_dtype = guess_proto_dtype(dtype)
value_info = helper.make_tensor_value_info(
name, proto_dtype, shape)
elif name in known_shapes:
info = known_shapes[name].tensor_type
proto_dtype = info.elem_type
if proto_dtype == 0:
value_info = helper.ValueInfoProto()
value_info.name = name
else:
shape = [getattr(d, 'dim_value', None) for d in info.shape.dim]
if len(shape) == 0:
shape = None
else:
shape = [None if s == 0 else s for s in shape]
value_info = helper.make_tensor_value_info(
name, proto_dtype, shape)
else:
value_info = helper.ValueInfoProto()
value_info.name = name
var_out.append(value_info)
if verbose > 0 and fLOG is not None: # pragma: no cover
fLOG("[select_model_inputs_outputs] nodes %r --> %r" % (
len(model.graph.node), len(keep_nodes)))
fLOG("[select_model_inputs_outputs] inputs: %r" % var_in)
fLOG("[select_model_inputs_outputs] inputs: %r" % var_out)
graph = helper.make_graph(keep_nodes, model.graph.name, var_in,
var_out, model.graph.initializer)
onnx_model = helper.make_model(graph)
onnx_model.ir_version = model.ir_version
onnx_model.producer_name = model.producer_name
onnx_model.producer_version = model.producer_version
onnx_model.domain = model.domain
onnx_model.model_version = model.model_version
onnx_model.doc_string = model.doc_string
if len(model.metadata_props) > 0: # pragma: no cover
values = {p.key: p.value for p in model.metadata_props}
helper.set_model_props(onnx_model, values)
del onnx_model.opset_import[:] # pylint: disable=E1101
for oimp in model.opset_import:
op_set = onnx_model.opset_import.add() # pylint: disable=E1101
op_set.domain = oimp.domain
op_set.version = oimp.version
# remove unused nodes
if remove_unused:
onnx_model = onnx_remove_node_unused(onnx_model, recursive=False)
return onnx_model
#outputs = sess.run([], {input1: img_preprocess})
# Step B
# get the output of block 4 add and relu operation for each add layer
node_count = len(model.graph.node)
model_node = str(model.graph.node)
# Step C
# find the name first
unit1_relu_name = 'SecondStageFeatureExtractor/resnet_v1_101/block4/unit_1/bottleneck_v1/Relu:0'
unit2_relu_name = 'SecondStageFeatureExtractor/resnet_v1_101/block4/unit_2/bottleneck_v1/Relu:0'
unit3_relu_name = 'SecondStageFeatureExtractor/resnet_v1_101/block4/unit_3/bottleneck_v1/Relu:0'
# Step D
# add into the current onnx model and save to a new model
info_unit1_relu_name = helper.ValueInfoProto()
info_unit1_relu_name.name = unit1_relu_name
info_unit2_relu_name = helper.ValueInfoProto()
info_unit2_relu_name.name = unit2_relu_name
info_unit3_relu_name = helper.ValueInfoProto()
info_unit3_relu_name.name = unit3_relu_name
# Step E
model.graph.output.extend([info_unit1_relu_name, info_unit2_relu_name, info_unit3_relu_name])
# Step F
onnx.save(model, os.path.join(path,'frozen_out.onnx'))
# Step G
# load the modified model with the intermediate output that you want to check
sess = rt.InferenceSession(os.path.join(path,'frozen_out.onnx'))
