def _make_value_info(self, variable): value_info = helper.ValueInfoProto() value_info.name = variable.full_name value_info.type.CopyFrom(variable.type.to_onnx_type()) if variable.type.doc_string: value_info.doc_string = variable.type.doc_string return value_info
def add_fmaps_to_outputs(self): model_nodes = self.onnx_model.graph.node model_outputs = [n.name for n in self.onnx_model.graph.output] for node in model_nodes: if node.output[0] in model_outputs: continue logging.info("{} -> {} ({})".format(node.name, node.output, len(node.output))) intermediate_layer_value_info = helper.ValueInfoProto() intermediate_layer_value_info.name = node.output[0] self.onnx_model.graph.output.append(intermediate_layer_value_info) onnx.save(self.onnx_model, self.model_path_extended)
def get_onnx_output(model, input, intermediate_node=None): sess = onnxruntime.InferenceSession(file_path) x = input x = x.astype(np.float32) input_name = model.graph.input[0].name if (intermediate_node != None): intermediate_layer_value_info = helper.ValueInfoProto() intermediate_layer_value_info.name = sys.argv[2] model.graph.output.extend([intermediate_layer_value_info]) onnx.save(model, file_path + '_1') sess = onnxruntime.InferenceSession(file_path + '_1') pred = sess.run([intermediate_layer_value_info.name], {input_name: x}) return pred pred = sess.run(None, {input_name: x}) return pred
def main(): # First read the ONNX file if (len(sys.argv) < 2): print("TF python file unspecified.", file=sys.stderr) exit(1) file_name = sys.argv[1] file_path = 'models/' + file_name model_name = file_name[:-5] # name without the '.onnx' extension model = onnx.load(file_path) model = preprocess_for_tf(model) x = np.load('debug/' + model_name + '/' + model_name + '_input.npy') x = x.astype(np.float32) input_name = model.graph.input[0].name output_name = model.graph.output[0].name if (len(sys.argv) > 2): intermediate_layer_value_info = helper.ValueInfoProto() intermediate_layer_value_info_name = 'tf_' + sys.argv[2] intermediate_layer_value_info = helper.make_tensor_value_info( intermediate_layer_value_info_name, TensorProto.FLOAT, []) model.graph.output.extend([intermediate_layer_value_info]) output = prepare(model).run(x) pred = getattr(output, intermediate_layer_value_info_name) np.save('debug/' + model_name + '/' + model_name + '_debug', pred) with open('debug/onnx_debug.txt', 'w') as f: f.write(common.numpy_float_array_to_float_val_str(pred)) print("Saving the onnx runtime intermediate output for " + intermediate_layer_value_info.name) exit() output = prepare(model).run(x) pred = getattr(output, output_name) np.save('debug/' + model_name + '/' + model_name + '_output', pred) with open('debug/onnx_output.txt', 'w') as f: f.write(common.numpy_float_array_to_float_val_str(pred)) output_dims = common.proto_val_to_dimension_tuple(model.graph.output[0]) print("Saving the onnx runtime output of dimension " + str(output_dims))
print("TF python file unspecified.", file=sys.stderr) exit(1) file_name = sys.argv[1] file_path = 'models/' + file_name model_name = file_name[:-5] # name without the '.onnx' extension model = onnx.load(file_path) sess = onnxruntime.InferenceSession(file_path) x = np.load('debug/' + model_name + '/' + model_name + '_input.npy') x = x.astype(np.float32) input_name = model.graph.input[0].name if (len(sys.argv) > 2): intermediate_layer_value_info = helper.ValueInfoProto() intermediate_layer_value_info.name = sys.argv[2] model.graph.output.extend([intermediate_layer_value_info]) onnx.save(model, file_path + '_1') sess = onnxruntime.InferenceSession(file_path + '_1') pred = sess.run([intermediate_layer_value_info.name], {input_name: x}) np.save('debug/' + model_name + '/' + model_name + '_debug', pred) with open('debug/onnx_debug.txt', 'w') as f: f.write(common.numpy_float_array_to_float_val_str(pred)) print("Saving the onnx runtime intermediate output for " + intermediate_layer_value_info.name) exit() pred = sess.run(None, {input_name: x}) np.save('debug/' + model_name + '/' + model_name + '_output', pred) with open('debug/onnx_output.txt', 'w') as f: f.write(common.numpy_float_array_to_float_val_str(pred))
def select_model_inputs_outputs(model, outputs=None, inputs=None): """ Takes a model and changes its outputs. @param model :epkg:`ONNX` model @param inputs new inputs, same ones if None @param outputs new outputs, same ones if None @return modified model The function removes unneeded files. """ if inputs is not None: raise NotImplementedError("Parameter inputs cannot be empty.") if outputs is None: raise RuntimeError("Parameter outputs cannot be None.") if not isinstance(outputs, list): outputs = [outputs] mark_var = {} for out in enumerate_model_node_outputs(model): mark_var[out] = 0 for inp in model.graph.input: mark_var[inp.name] = 0 for out in outputs: if out not in mark_var: raise ValueError("Output '{}' not found in model.".format(out)) mark_var[out] = 1 nodes = model.graph.node[::-1] mark_op = {} for node in nodes: mark_op[node.name] = 0 # We mark all the nodes we need to keep. nb = 1 while nb > 0: nb = 0 for node in nodes: if mark_op[node.name] == 1: continue mod = False for out in node.output: if mark_var[out] == 1: mark_op[node.name] = 1 mod = True break if not mod: continue nb += 1 for inp in node.input: if mark_var.get(inp, 0) == 1: continue mark_var[inp] = 1 nb += 1 # All nodes verifies mark_op[node.name] == 1 keep_nodes = [node for node in nodes if mark_op[node.name] == 1] var_out = [] for out in outputs: value_info = helper.ValueInfoProto() value_info.name = out var_out.append(value_info) graph = helper.make_graph(keep_nodes, model.graph.name, model.graph.input, var_out, model.graph.initializer) onnx_model = helper.make_model(graph) onnx_model.ir_version = model.ir_version onnx_model.producer_name = model.producer_name onnx_model.producer_version = model.producer_version onnx_model.domain = model.domain onnx_model.model_version = model.model_version onnx_model.doc_string = model.doc_string if len(onnx_model.graph.input) != len(model.graph.input): # pylint: disable=E1101 raise RuntimeError("Input mismatch {} != {}".format( len(onnx_model.input), len(model.input))) # pylint: disable=E1101 return onnx_model
def augment_graph(self, activation_only=False, output_only=False): ''' Adds nodes to all quantization_candidates op type nodes in model and ensures their outputs are stored as part of the graph output :param activation_only(bool): whether to dump activation tensor only :param output_only(bool): whether to dump output_only :return: augmented ONNX model ''' model = copy.deepcopy(self.model) model_nodes_names = [node.name for node in model.graph.node] added_nodes = [] added_outputs = [] tensors_to_dump = set() for augment_node_type in self.augment_nodes: if augment_node_type not in [ 'ReduceMin', 'ReduceMax', 'DequantizeLinear' ]: raise ValueError("Unexpected augment_node {} only \ ReduceMin/ReduceMax are supported".format( augment_node_type)) if self.already_quantized: # mapping between fp32 node and int8 node new_white_nodes = [] for white_node in self.white_nodes: new_white_node = white_node + "_quant" assert new_white_node in model_nodes_names, "no quantized {} \ in the graph".format( white_node) new_white_nodes.append(new_white_node) self.white_nodes = new_white_nodes initializer_names = [i.name for i in model.graph.initializer] for node in model.graph.node: # pylint: disable=no-member should_be_dump = ((node.op_type in self.dump_op_types) and (node.name not in self.black_nodes)) or \ (node.name in self.white_nodes) if should_be_dump: if not output_only: if node.op_type == "Attention": if len(node.input) >= 3: logger.debug( "indice input {} of attention node {} is integer" .format(node.input[3:], node.name)) tensors_to_dump.update(node.input[:2]) else: tensors_to_dump.update(node.input) elif node.op_type == "Gather": logger.debug( "indice input {} of gather node {} is integer". format(node.input[-1], node.name)) tensors_to_dump.update(node.input[:-1]) else: tensors_to_dump.update(node.input) else: for input in node.input: if input in initializer_names: tensors_to_dump.add(input) tensors_to_dump.update(node.output) tensors_tmp = set() if activation_only: for tensor in tensors_to_dump: if tensor not in initializer_names: # pylint: disable=no-member tensors_tmp.add(tensor) tensors_to_dump = tensors_tmp for tensor in tensors_to_dump: if self.augment_nodes: for augment_node_type in self.augment_nodes: if augment_node_type in ['ReduceMin', 'ReduceMax']: # dump tensor for calibration augment_node_name = tensor + "_" + augment_node_type augment_node = onnx.helper.make_node( augment_node_type, [tensor], [augment_node_name], augment_node_name, keepdims=0) added_nodes.append(augment_node) added_outputs.append( helper.make_tensor_value_info( augment_node.output[0], # pylint: disable=no-member TensorProto.FLOAT, ())) # pylint: disable=no-member else: # insert DequantizeLinear node as output augment_node_name = tensor + "_new_" + augment_node_type scale, zero_point = self.model_wrapper.get_scale_zo( tensor) if scale: # the tensor is in INT8 dtype nodes, output = self._dequantize( tensor, scale, zero_point) added_nodes.extend(nodes) added_outputs.append( helper.make_tensor_value_info( output, # pylint: disable=no-member TensorProto.FLOAT, ())) # pylint: disable=no-member else: # the tensor is in FP32 dtype if tensor not in [ t.name for t in model.graph.output ]: added_tensor = helper.ValueInfoProto() added_tensor.name = tensor added_outputs.append(added_tensor) else: if tensor not in [t.name for t in model.graph.output]: added_tensor = helper.ValueInfoProto() added_tensor.name = tensor added_outputs.append(added_tensor) if self.augment_nodes: model.graph.node.extend(added_nodes) # pylint: disable=no-member model.graph.output.extend(added_outputs) # pylint: disable=no-member self.augmented_model = model onnx.save(model, self.augmented_model_path)
def main(argv): parser = argparse.ArgumentParser() parser.add_argument( "--input_file", help="Input npz or jpg file." ) parser.add_argument( "--output_file", help="Output npz" ) parser.add_argument( "--model_path", help="onnx model path." ) parser.add_argument( "--dump_tensors", help="Dump all output tensors into a file in npz format" ) parser.add_argument( "--mean", default='0,0,0', help="Per Channel image mean values" ) parser.add_argument( "--mean_file", default='mean', help="Data set image mean of [Channels x Height x Width] dimensions " + "(numpy array). Set to '' for no mean subtraction." ) parser.add_argument( "--std", help="Per Channel image std values", default='1,1,1' ) parser.add_argument( "--input_scale", type=float, default=1.0, help="Multiply input features by this scale to finish preprocessing." ) parser.add_argument( "--raw_scale", type=float, default=255.0, help="Multiply raw input by this scale before preprocessing." ) parser.add_argument( "--net_input_dims", default='224,224', help="'height,width' dimensions of network input spatial dimensions." ) parser.add_argument( "--image_resize_dims", default='256,256', help="To resize to this size first, then crop to net_input_dims." ) args = parser.parse_args() # preprocessor = cvi_preprocess() # preprocessor.config(net_input_dims=args.net_input_dims, # resize_dims=args.image_resize_dims, # mean=args.mean, # mean_file=args.mean_file, # input_scale=args.input_scale, # raw_scale=args.raw_scale, # std=args.std) input=None file_extension = args.input_file.split(".")[-1].lower() if file_extension == "jpg": mean = [float(x) for x in args.mean.split(",")] if args.std: std = [float(x) for x in args.std.split(",")] else: std = [1, 1, 1] net_input_dims = [int(x) for x in args.net_input_dims.split(",")] if args.image_resize_dims: image_resize_dims = [int(x) for x in args.image_resize_dims.split(",")] else: image_resize_dims = net_input_dims input = preprocessor.run(args.input_file) elif file_extension == "npz": input = np.load(args.input_file)['input'] else: print("Not support {} extension") ort_outs = inference(input, args.model_path) np.savez(args.output_file, **{'output': ort_outs[0]}) #print("org ort_outs", ort_outs) if args.dump_tensors: # second pass for dump all output # plz refre https://github.com/microsoft/onnxruntime/issues/1455 output_keys = ['output'] model = onnx.load(args.model_path) # tested commited #c3cea486d https://github.com/microsoft/onnxruntime.git for x in model.graph.node: _intermediate_tensor_name = list(x.output) intermediate_tensor_name = ",".join(_intermediate_tensor_name) intermediate_layer_value_info = helper.ValueInfoProto() intermediate_layer_value_info.name = intermediate_tensor_name model.graph.output.append(intermediate_layer_value_info) output_keys.append(intermediate_layer_value_info.name + '_' + x.op_type) model_file = args.model_path.split("/")[-1] dump_all_onnx = "all_{}".format(model_file) if not os.path.exists(dump_all_onnx): onnx.save(model, dump_all_onnx) else: print("{} is exitsed!".format(dump_all_onnx)) print("dump multi-output onnx all tensor at ", dump_all_onnx) # dump all inferneced tensor ort_outs = inference(input, dump_all_onnx) tensor_all_dict = dict(zip(output_keys, map(np.ndarray.flatten, ort_outs))) tensor_all_dict['input'] = input np.savez(args.dump_tensors, **tensor_all_dict) print("dump all tensor at ", args.dump_tensors)
def augment_graph(self): ''' Adds nodes to all quantization_candidates op type nodes in model and ensures their outputs are stored as part of the graph output :return: augmented ONNX model ''' model = copy.deepcopy(self.model) added_nodes = [] added_outputs = [] tensors_to_dump = set() for node in model.graph.node: # pylint: disable=no-member should_be_dump = ((node.op_type in self.dump_op_types) and (node.name not in self.black_nodes)) or \ (node.name in self.white_nodes) if should_be_dump: if node.op_type == "Attention": if len(node.input) >= 3: logger.debug("indice input {} of attention node {} is in integer format" .format(node.input[3:], node.name)) tensors_to_dump.update(node.input[:2]) else: tensors_to_dump.update(node.input) elif node.op_type == "Gather": logger.debug("indice input {} of gather node {} is in integer format" .format(node.input[-1], node.name)) tensors_to_dump.update(node.input[:-1]) else: tensors_to_dump.update(node.input) tensors_to_dump.update(node.output) for tensor in tensors_to_dump: if tensor in model.graph.initializer: # pylint: disable=no-member tensors_to_dump.remove(tensor) for tensor in tensors_to_dump: if self.augment_nodes: for augment_node_type in self.augment_nodes: if augment_node_type not in ['ReduceMin', 'ReduceMax']: raise ValueError("Unexpected augment_node {} only \ ReduceMin/ReduceMax are supported".format(augment_node_type)) augment_node_name = tensor + "_" + augment_node_type augment_node = onnx.helper.make_node(augment_node_type, [tensor], [augment_node_name], augment_node_name, keepdims=0) added_nodes.append(augment_node) added_outputs.append(helper.make_tensor_value_info( augment_node.output[0], # pylint: disable=no-member TensorProto.FLOAT, ())) # pylint: disable=no-member else: if tensor not in [t.name for t in model.graph.output]: added_tensor = helper.ValueInfoProto() added_tensor.name = tensor added_outputs.append(added_tensor) if self.augment_nodes: model.graph.node.extend(added_nodes) # pylint: disable=no-member model.graph.output.extend(added_outputs) # pylint: disable=no-member self.augmented_model = model onnx.save(model, self.augmented_model_path)
traceback.format_exc() ]) + '\n\n\n') tested_file.flush() continue if dataset == 'cifar10': input = np.array(test_input, dtype=np.float32).reshape([1, 32, 32, 3]) elif dataset == 'mnist': input = np.array(test_input, dtype=np.float32).reshape([1, 28, 28, 1]) if is_onnx: input = input.transpose(0, 3, 1, 2) for name, shape in output_info: out_node = helper.ValueInfoProto(type=helper.TypeProto()) out_node.name = name out_node.type.tensor_type.elem_type = model.graph.output[ 0].type.tensor_type.elem_type if len(shape) == 4: shape = [shape[0], shape[3], shape[1], shape[2]] for dim_value in shape: dim = out_node.type.tensor_type.shape.dim.add() dim.dim_value = dim_value model.graph.output.append(out_node) runnable = rt.prepare(model, 'CPU') pred = runnable.run(input) #print(pred) else: if not (is_saved_tf_model or is_pb_file): input = np.array(test_input, dtype=np.float32)
def select_model_inputs_outputs(model, outputs=None, inputs=None, infer_shapes=False, overwrite=None, remove_unused=True, verbose=0, fLOG=None): """ Takes a model and changes its outputs. :param model: :epkg:`ONNX` model :param inputs: new inputs, same ones if None :param outputs: new outputs, same ones if None :param infer_shapes: infer inputs and outputs shapes :param overwrite: overwrite type and shapes for inputs or outputs, *overwrite* is a dictionary `{'name': (numpy dtype, shape)}` :param remove_unused: remove unused nodes from the graph :param verbose: display information while converting :param fLOG: logging function :return: modified model The function removes unneeded nodes. .. exref:: :title: Change ONNX model inputs The following exampels shows how to change the inputs of model to bypass the first nodes. Shape inferences fails to determine the new inputs type. They need to be overwritten. `verbose=1, fLOG=print` shows the number of deleted nodes. :: import onnx from mlprodict.onnx_tools.onnx_manipulations import select_model_inputs_outputs onx = onnx.load(path) onx2 = select_model_inputs_outputs( onx, inputs=["SentenceTokenizer/SentencepieceTokenizeOp:0", "SentenceTokenizer/SentencepieceTokenizeOp:1"], infer_shapes=True, verbose=1, fLOG=print, overwrite={'SentenceTokenizer/SentencepieceTokenizeOp:0': (numpy.int32, None), 'SentenceTokenizer/SentencepieceTokenizeOp:1': (numpy.int64, None)}) onnx.save(onx2, path2) .. versionchanged:: 0.6 Supports the case where inputs are changed. .. versionchanged:: 0.7 Parameter *remove_unused* was added. Unused are removed by default. """ if inputs is not None and not isinstance(inputs, list): inputs = [inputs] if outputs is not None and not isinstance(outputs, list): outputs = [outputs] if inputs is None: inputs = [i.name for i in model.graph.input] if outputs is None: outputs = [o.name for o in model.graph.output] mark_var = {} for out in enumerate_model_node_outputs(model): mark_var[out] = 0 for inp in inputs: mark_var[inp] = 0 for out in outputs: if out not in mark_var: raise ValueError( # pragma: no cover "Output '{}' not found in model.".format(out)) mark_var[out] = 1 nodes = model.graph.node[::-1] mark_op = {} for node in nodes: mark_op[node.name] = 0 # We mark all the nodes we need to keep. nb = 1 while nb > 0: nb = 0 for node in nodes: if mark_op[node.name] == 1: continue mod = False for out in node.output: if mark_var[out] == 1: mark_op[node.name] = 1 mod = True break if not mod: continue nb += 1 for inp in node.input: if inp in inputs: continue if mark_var.get(inp, 0) == 1: continue mark_var[inp] = 1 nb += 1 # All nodes verifies mark_op[node.name] == 1 keep_nodes = [node for node in nodes if mark_op[node.name] == 1] known_shapes = {} if infer_shapes: shapes = shape_inference.infer_shapes(model) for shape in shapes.graph.value_info: # pylint: disable=E1101 known_shapes[shape.name] = shape.type for shape in shapes.graph.input: # pylint: disable=E1101 known_shapes[shape.name] = shape.type for shape in shapes.graph.output: # pylint: disable=E1101 known_shapes[shape.name] = shape.type else: for shape in model.graph.input: # pylint: disable=E1101 known_shapes[shape.name] = shape.type for shape in model.graph.output: # pylint: disable=E1101 known_shapes[shape.name] = shape.type var_in = [] for name in inputs: if overwrite is not None and name in overwrite: dtype, shape = overwrite[name] proto_dtype = guess_proto_dtype(dtype) value_info = helper.make_tensor_value_info( name, proto_dtype, shape) elif name in known_shapes: info = known_shapes[name].tensor_type proto_dtype = info.elem_type if proto_dtype == 0: value_info = helper.ValueInfoProto() value_info.name = name else: shape = [getattr(d, 'dim_value', None) for d in info.shape.dim] if len(shape) == 0: shape = None else: shape = [None if s == 0 else s for s in shape] value_info = helper.make_tensor_value_info( name, proto_dtype, shape) else: value_info = helper.ValueInfoProto() value_info.name = name var_in.append(value_info) var_out = [] for name in outputs: if overwrite is not None and name in overwrite: dtype, shape = overwrite[name] proto_dtype = guess_proto_dtype(dtype) value_info = helper.make_tensor_value_info( name, proto_dtype, shape) elif name in known_shapes: info = known_shapes[name].tensor_type proto_dtype = info.elem_type if proto_dtype == 0: value_info = helper.ValueInfoProto() value_info.name = name else: shape = [getattr(d, 'dim_value', None) for d in info.shape.dim] if len(shape) == 0: shape = None else: shape = [None if s == 0 else s for s in shape] value_info = helper.make_tensor_value_info( name, proto_dtype, shape) else: value_info = helper.ValueInfoProto() value_info.name = name var_out.append(value_info) if verbose > 0 and fLOG is not None: # pragma: no cover fLOG("[select_model_inputs_outputs] nodes %r --> %r" % ( len(model.graph.node), len(keep_nodes))) fLOG("[select_model_inputs_outputs] inputs: %r" % var_in) fLOG("[select_model_inputs_outputs] inputs: %r" % var_out) graph = helper.make_graph(keep_nodes, model.graph.name, var_in, var_out, model.graph.initializer) onnx_model = helper.make_model(graph) onnx_model.ir_version = model.ir_version onnx_model.producer_name = model.producer_name onnx_model.producer_version = model.producer_version onnx_model.domain = model.domain onnx_model.model_version = model.model_version onnx_model.doc_string = model.doc_string if len(model.metadata_props) > 0: # pragma: no cover values = {p.key: p.value for p in model.metadata_props} helper.set_model_props(onnx_model, values) del onnx_model.opset_import[:] # pylint: disable=E1101 for oimp in model.opset_import: op_set = onnx_model.opset_import.add() # pylint: disable=E1101 op_set.domain = oimp.domain op_set.version = oimp.version # remove unused nodes if remove_unused: onnx_model = onnx_remove_node_unused(onnx_model, recursive=False) return onnx_model
#outputs = sess.run([], {input1: img_preprocess}) # Step B # get the output of block 4 add and relu operation for each add layer node_count = len(model.graph.node) model_node = str(model.graph.node) # Step C # find the name first unit1_relu_name = 'SecondStageFeatureExtractor/resnet_v1_101/block4/unit_1/bottleneck_v1/Relu:0' unit2_relu_name = 'SecondStageFeatureExtractor/resnet_v1_101/block4/unit_2/bottleneck_v1/Relu:0' unit3_relu_name = 'SecondStageFeatureExtractor/resnet_v1_101/block4/unit_3/bottleneck_v1/Relu:0' # Step D # add into the current onnx model and save to a new model info_unit1_relu_name = helper.ValueInfoProto() info_unit1_relu_name.name = unit1_relu_name info_unit2_relu_name = helper.ValueInfoProto() info_unit2_relu_name.name = unit2_relu_name info_unit3_relu_name = helper.ValueInfoProto() info_unit3_relu_name.name = unit3_relu_name # Step E model.graph.output.extend([info_unit1_relu_name, info_unit2_relu_name, info_unit3_relu_name]) # Step F onnx.save(model, os.path.join(path,'frozen_out.onnx')) # Step G # load the modified model with the intermediate output that you want to check sess = rt.InferenceSession(os.path.join(path,'frozen_out.onnx'))