def test_onnx_remove_unused_outputs(self): dtype = numpy.float32 x = numpy.array([1, 2, 4, 5, 5, 4]).astype(numpy.float32).reshape( (3, 2)) cop = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=TARGET_OPSET) cop2 = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=TARGET_OPSET) cop3 = OnnxAdd('X', numpy.array([2], dtype=dtype), op_version=TARGET_OPSET, output_names=['inter']) cop4 = OnnxSub(OnnxMul(cop, cop3, op_version=TARGET_OPSET), cop2, output_names=['final'], op_version=TARGET_OPSET) model_def = cop4.to_onnx({'X': x}) model_def = select_model_inputs_outputs(model_def, "inter", infer_shapes=True, remove_unused=False) stats = onnx_statistics(model_def, optim=True) c1 = model_def.SerializeToString() new_model = onnx_remove_node_unused(model_def) c2 = model_def.SerializeToString() self.assertEqual(c1, c2) stats2 = onnx_statistics(model_def, optim=True) stats3 = onnx_statistics(new_model, optim=False) self.assertEqual(stats['ninits'], 2) self.assertEqual(stats2['ninits'], 2) self.assertEqual(stats3['ninits'], 1) self.assertEqual(stats2['nnodes'], 1) self.assertEqual(stats3['nnodes'], 1) oinf1 = OnnxInference(model_def) y1 = oinf1.run({'X': x}) oinf2 = OnnxInference(new_model) y2 = oinf2.run({'X': x}) self.assertNotIn('final', y1) self.assertNotIn('final', y2) self.assertIn('inter', y1) self.assertIn('inter', y2) self.assertEqualArray(y1['inter'], y2['inter'])
def add_loss_output(onx, score_name='squared_error', loss_name='loss', label_name='label', weight_name=None, penalty=None, output_index=None, **kwargs): """ Modifies an ONNX graph to add operators to score and allow training. :param onx: onx graph :param score_name: name of the score :param loss_name: name of the output loss :param label_name: name of the label input :param weight_name: None or any value to consider weight while computing loss :param penalty: dictionary similar to the following one `{ weight_name: {'l1': alpha, 'l2': beta} }` or `{ weight_name: beta}`, it adds a L1 and/or L2 penalty to one input or initializer, penalty = :math:`|w| \\alpha + w^2 \\beta` :param output_index: the output used to compute the loss, if None, the function assumes there is only one output, it must be specified if there are more than 1, it can be an integer or a string (output name) :param kwargs: additional arguments for losses (see below) :return: modified graph Possible values for *score_name*: * `'squared_error'` or `'l2`': :math:`\\sum_i{(f(x_i)-y_i)^2}` or :math:`\\sum_i{w_i (f(x_i)-y_i)^2}` if *weight_name* is not None * `'absolute_error'` or `'l1`': :math:`\\sum_i{|f(x_i)-y_i|}` or :math:`\\sum_i{w_i |f(x_i)-y_i|}` if *weight_name* is not None * `'elastic'`: mixture of losses, kwargs must define *l1_weight* and *l2_weight*, undefined, default value are 0.5 * `'log'`: log loss :math:`(1-yt)\\log(1-yp) - yt\\log(yp)`, this only works for a binary classification where *yp* is the predicted probability, *yt* is the expected probability. *yt* is expected to be binary, *yp* is a matrix with two columns, the sum on every line is 1. See example :ref:`l-orttraining-nn-gpu`. Next example shows the loss with L1 and L2 loss. .. gdot:: :script: DOT-SECTION import numpy from sklearn.datasets import make_regression from sklearn.model_selection import train_test_split from sklearn.linear_model import LinearRegression from mlprodict.onnx_conv import to_onnx from mlprodict.onnxrt import OnnxInference from onnxcustom import __max_supported_opset__ as opset from onnxcustom.utils.orttraining_helper import add_loss_output from onnxcustom.training.optimizers import OrtGradientOptimizer X, y = make_regression( # pylint: disable=W0632 100, n_features=10, bias=2, random_state=0) X = X.astype(numpy.float32) y = y.astype(numpy.float32) w = (numpy.random.rand(y.shape[0]) + 1).astype(X.dtype) X_train, _, y_train, __, w_train, ___ = train_test_split(X, y, w) reg = LinearRegression() reg.fit(X_train, y_train, sample_weight=w_train) reg.coef_ = reg.coef_.reshape((1, -1)) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}) onx_loss = add_loss_output( onx, weight_name='weight', score_name='elastic', l1_weight=0.1, l2_weight=0.9) print("DOT-SECTION", OnnxInference(onx_loss).to_dot()) Next example shows how to add a L2 loss with L1 and L2 penalties on the coefficients. .. gdot:: :script: DOT-SECTION import numpy from sklearn.datasets import make_regression from sklearn.model_selection import train_test_split from sklearn.linear_model import LinearRegression from mlprodict.onnx_conv import to_onnx from mlprodict.onnxrt import OnnxInference from onnxcustom import __max_supported_opset__ as opset from onnxcustom.utils.orttraining_helper import add_loss_output from onnxcustom.training.optimizers import OrtGradientOptimizer X, y = make_regression( # pylint: disable=W0632 100, n_features=10, bias=2, random_state=0) X = X.astype(numpy.float32) y = y.astype(numpy.float32) w = (numpy.random.rand(y.shape[0]) + 1).astype(X.dtype) X_train, _, y_train, __, w_train, ___ = train_test_split(X, y, w) reg = LinearRegression() reg.fit(X_train, y_train, sample_weight=w_train) reg.coef_ = reg.coef_.reshape((1, -1)) onx = to_onnx(reg, X_train, target_opset=opset, black_op={'LinearRegressor'}) onx_loss = add_loss_output( onx, weight_name='weight', score_name='elastic', penalty={'coef': {'l1': 0.5, 'l2':0.5}, 'intercept': {'l1': 0.5, 'l2':0.5}}) print("DOT-SECTION", OnnxInference(onx_loss).to_dot()) """ from mlprodict.onnx_tools.optim import onnx_remove_node_unused # rename every intermediate output call label def _replace(ens): for i in range(len(ens)): # pylint: disable=C0200 if ens[i] == 'label': ens[i] = '_label_' for node in onx.graph.node: if "_label_" in node.input or "_label_" in node.output: raise RuntimeError( # pragma: no cover "One intermediate result contains '_label_'. " "It should be removed manually.\n%r" % node) _replace(node.input) _replace(node.output) if output_index is None: if len(onx.graph.output) != 1: raise ValueError( # pragma: no cover "Unable to guess the output to compare to the " "expacted labels among %r." % ([o.name for o in onx.graph.output])) outputs = onx.graph.output output_index = 0 elif isinstance(output_index, int): outputs = [onx.graph.output[output_index]] elif isinstance(output_index, str): outputs = [(i, o) for i, o in enumerate(onx.graph.output) if o.name == output_index] if len(outputs) != 1: raise ValueError( # pragma: no cover "Unable to find output %r in %r." % (output_index, [o.name for o in onx.graph.output])) output_index = outputs[0][0] outputs = [outputs[0][1]] else: raise TypeError( # pragma: no cover f"output_index must be an integer or a str not {type(output_index)!r}." ) existing_names = [] for node in onx.graph.node: existing_names.extend(node.output) existing_names.extend(node.input) existing_names = set(existing_names) output_onx = onx.graph.output[output_index] output_name = output_onx.name elem = output_onx.type.tensor_type.elem_type if elem == 0: raise TypeError( # pragma: no cover f"Unable to guess input tensor type from {output_onx!r}.") shape = [] for d in output_onx.type.tensor_type.shape.dim: shape.append(d.dim_value if d.dim_value > 0 else None) if score_name in ('squared_error', 'l2'): inits, inputs, nodes, outputs = _loss_l2(existing_names, elem, shape, output_name, label_name, weight_name, loss_name) elif score_name in ('absolute_error', 'l1'): inits, inputs, nodes, outputs = _loss_l1(existing_names, elem, shape, output_name, label_name, weight_name, loss_name) elif score_name == 'elastic': inits, inputs, nodes, outputs = _loss_elastic(existing_names, elem, shape, output_name, label_name, weight_name, loss_name, **kwargs) elif score_name == 'log': shape = (None, 1) inits, inputs, nodes, outputs = _loss_log(existing_names, elem, shape, output_name, label_name, weight_name, loss_name, **kwargs) else: raise NotImplementedError( # pragma: no cover f"Unexpected {score_name!r} value for score_name.") if penalty is not None: final_name = nodes[-1].output[0] loss_name = _unique_name(existing_names, "loss_diff") nodes[-1].output[0] = loss_name names = [] for k, v in penalty.items(): if isinstance(v, float): v = {'l2': v} inits_to_add, nodes_to_add = penalty_loss_onnx( k, dtype=TENSOR_TYPE_TO_NP_TYPE[elem], existing_names=existing_names, **v) names.append(nodes_to_add[-1].output[0]) nodes.extend(nodes_to_add) inits.extend(inits_to_add) # Operator Sum does not have a gradient. if len(names) == 1: pen_name = names[0] else: current = names[0] for i in range(1, len(names)): new_name = _unique_name(existing_names, "sumop") nodes.append(make_node('Add', [current, names[i]], [new_name])) current = new_name pen_name = current cst_shape = _unique_name(existing_names, "shapevect") inits.append( from_array(numpy.array([-1, 1], dtype=numpy.int64), name=cst_shape)) loss_reshape = _unique_name(existing_names, "loss_reshape") pen_reshape = _unique_name(existing_names, "penalty_reshape") nodes.extend([ make_node("Reshape", [pen_name, cst_shape], [pen_reshape]), make_node("Reshape", [loss_name, cst_shape], [loss_reshape]) ]) nodes.append( make_node('Add', [pen_reshape, loss_reshape], [final_name])) inits = list(onx.graph.initializer) + inits graph = make_graph( list(onx.graph.node) + nodes, onx.graph.name, list(onx.graph.input) + inputs, outputs + [onx.graph.output[output_index]], inits) onnx_model = make_model(graph) onnx_model.ir_version = onx.ir_version onnx_model.producer_name = onx.producer_name onnx_model.producer_version = onx.producer_version onnx_model.domain = onx.domain onnx_model.model_version = onx.model_version onnx_model.doc_string = onx.doc_string if len(onx.metadata_props) > 0: values = {p.key: p.value for p in onx.metadata_props} set_model_props(onnx_model, values) # fix opset import del onnx_model.opset_import[:] # pylint: disable=E1101 for oimp in onx.opset_import: op_set = onnx_model.opset_import.add() # pylint: disable=E1101 op_set.domain = oimp.domain op_set.version = oimp.version return _rewrite_op_no_grad(onnx_remove_node_unused(onnx_model))