def test_onnx_remove_unused_outputs(self):
dtype = numpy.float32
x = numpy.array([1, 2, 4, 5, 5, 4]).astype(numpy.float32).reshape(
(3, 2))
cop = OnnxAdd('X',
numpy.array([1], dtype=dtype),
op_version=TARGET_OPSET)
cop2 = OnnxAdd('X',
numpy.array([1], dtype=dtype),
op_version=TARGET_OPSET)
cop3 = OnnxAdd('X',
numpy.array([2], dtype=dtype),
op_version=TARGET_OPSET,
output_names=['inter'])
cop4 = OnnxSub(OnnxMul(cop, cop3, op_version=TARGET_OPSET),
cop2,
output_names=['final'],
op_version=TARGET_OPSET)
model_def = cop4.to_onnx({'X': x})
model_def = select_model_inputs_outputs(model_def,
"inter",
infer_shapes=True,
remove_unused=False)
stats = onnx_statistics(model_def, optim=True)
c1 = model_def.SerializeToString()
new_model = onnx_remove_node_unused(model_def)
c2 = model_def.SerializeToString()
self.assertEqual(c1, c2)
stats2 = onnx_statistics(model_def, optim=True)
stats3 = onnx_statistics(new_model, optim=False)
self.assertEqual(stats['ninits'], 2)
self.assertEqual(stats2['ninits'], 2)
self.assertEqual(stats3['ninits'], 1)
self.assertEqual(stats2['nnodes'], 1)
self.assertEqual(stats3['nnodes'], 1)
oinf1 = OnnxInference(model_def)
y1 = oinf1.run({'X': x})
oinf2 = OnnxInference(new_model)
y2 = oinf2.run({'X': x})
self.assertNotIn('final', y1)
self.assertNotIn('final', y2)
self.assertIn('inter', y1)
self.assertIn('inter', y2)
self.assertEqualArray(y1['inter'], y2['inter'])
def add_loss_output(onx,
score_name='squared_error',
loss_name='loss',
label_name='label',
weight_name=None,
penalty=None,
output_index=None,
**kwargs):
"""
Modifies an ONNX graph to add operators to score and allow training.
:param onx: onx graph
:param score_name: name of the score
:param loss_name: name of the output loss
:param label_name: name of the label input
:param weight_name: None or any value to consider weight
while computing loss
:param penalty: dictionary similar to the
following one `{ weight_name: {'l1': alpha, 'l2': beta} }`
or `{ weight_name: beta}`,
it adds a L1 and/or L2 penalty to one input or initializer,
penalty = :math:`|w| \\alpha + w^2 \\beta`
:param output_index: the output used to compute the loss,
if None, the function assumes there is only one output,
it must be specified if there are more than 1,
it can be an integer or a string (output name)
:param kwargs: additional arguments for losses (see below)
:return: modified graph
Possible values for *score_name*:
* `'squared_error'` or `'l2`': :math:`\\sum_i{(f(x_i)-y_i)^2}` or
:math:`\\sum_i{w_i (f(x_i)-y_i)^2}` if *weight_name*
is not None
* `'absolute_error'` or `'l1`': :math:`\\sum_i{|f(x_i)-y_i|}` or
:math:`\\sum_i{w_i |f(x_i)-y_i|}` if *weight_name*
is not None
* `'elastic'`: mixture of losses, kwargs must define
*l1_weight* and *l2_weight*, undefined, default value are 0.5
* `'log'`: log loss :math:`(1-yt)\\log(1-yp) - yt\\log(yp)`,
this only works for a binary classification where *yp* is the
predicted probability, *yt* is the expected probability.
*yt* is expected to be binary, *yp* is a matrix with two
columns, the sum on every line is 1.
See example :ref:`l-orttraining-nn-gpu`.
Next example shows the loss with L1 and L2 loss.
.. gdot::
:script: DOT-SECTION
import numpy
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from mlprodict.onnx_conv import to_onnx
from mlprodict.onnxrt import OnnxInference
from onnxcustom import __max_supported_opset__ as opset
from onnxcustom.utils.orttraining_helper import add_loss_output
from onnxcustom.training.optimizers import OrtGradientOptimizer
X, y = make_regression( # pylint: disable=W0632
100, n_features=10, bias=2, random_state=0)
X = X.astype(numpy.float32)
y = y.astype(numpy.float32)
w = (numpy.random.rand(y.shape[0]) + 1).astype(X.dtype)
X_train, _, y_train, __, w_train, ___ = train_test_split(X, y, w)
reg = LinearRegression()
reg.fit(X_train, y_train, sample_weight=w_train)
reg.coef_ = reg.coef_.reshape((1, -1))
onx = to_onnx(reg, X_train, target_opset=opset,
black_op={'LinearRegressor'})
onx_loss = add_loss_output(
onx, weight_name='weight', score_name='elastic',
l1_weight=0.1, l2_weight=0.9)
print("DOT-SECTION", OnnxInference(onx_loss).to_dot())
Next example shows how to add a L2 loss with L1 and L2 penalties
on the coefficients.
.. gdot::
:script: DOT-SECTION
import numpy
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from mlprodict.onnx_conv import to_onnx
from mlprodict.onnxrt import OnnxInference
from onnxcustom import __max_supported_opset__ as opset
from onnxcustom.utils.orttraining_helper import add_loss_output
from onnxcustom.training.optimizers import OrtGradientOptimizer
X, y = make_regression( # pylint: disable=W0632
100, n_features=10, bias=2, random_state=0)
X = X.astype(numpy.float32)
y = y.astype(numpy.float32)
w = (numpy.random.rand(y.shape[0]) + 1).astype(X.dtype)
X_train, _, y_train, __, w_train, ___ = train_test_split(X, y, w)
reg = LinearRegression()
reg.fit(X_train, y_train, sample_weight=w_train)
reg.coef_ = reg.coef_.reshape((1, -1))
onx = to_onnx(reg, X_train, target_opset=opset,
black_op={'LinearRegressor'})
onx_loss = add_loss_output(
onx, weight_name='weight', score_name='elastic',
penalty={'coef': {'l1': 0.5, 'l2':0.5},
'intercept': {'l1': 0.5, 'l2':0.5}})
print("DOT-SECTION", OnnxInference(onx_loss).to_dot())
"""
from mlprodict.onnx_tools.optim import onnx_remove_node_unused
# rename every intermediate output call label
def _replace(ens):
for i in range(len(ens)): # pylint: disable=C0200
if ens[i] == 'label':
ens[i] = '_label_'
for node in onx.graph.node:
if "_label_" in node.input or "_label_" in node.output:
raise RuntimeError( # pragma: no cover
"One intermediate result contains '_label_'. "
"It should be removed manually.\n%r" % node)
_replace(node.input)
_replace(node.output)
if output_index is None:
if len(onx.graph.output) != 1:
raise ValueError( # pragma: no cover
"Unable to guess the output to compare to the "
"expacted labels among %r." %
([o.name for o in onx.graph.output]))
outputs = onx.graph.output
output_index = 0
elif isinstance(output_index, int):
outputs = [onx.graph.output[output_index]]
elif isinstance(output_index, str):
outputs = [(i, o) for i, o in enumerate(onx.graph.output)
if o.name == output_index]
if len(outputs) != 1:
raise ValueError( # pragma: no cover
"Unable to find output %r in %r." %
(output_index, [o.name for o in onx.graph.output]))
output_index = outputs[0][0]
outputs = [outputs[0][1]]
else:
raise TypeError( # pragma: no cover
f"output_index must be an integer or a str not {type(output_index)!r}."
)
existing_names = []
for node in onx.graph.node:
existing_names.extend(node.output)
existing_names.extend(node.input)
existing_names = set(existing_names)
output_onx = onx.graph.output[output_index]
output_name = output_onx.name
elem = output_onx.type.tensor_type.elem_type
if elem == 0:
raise TypeError( # pragma: no cover
f"Unable to guess input tensor type from {output_onx!r}.")
shape = []
for d in output_onx.type.tensor_type.shape.dim:
shape.append(d.dim_value if d.dim_value > 0 else None)
if score_name in ('squared_error', 'l2'):
inits, inputs, nodes, outputs = _loss_l2(existing_names, elem, shape,
output_name, label_name,
weight_name, loss_name)
elif score_name in ('absolute_error', 'l1'):
inits, inputs, nodes, outputs = _loss_l1(existing_names, elem, shape,
output_name, label_name,
weight_name, loss_name)
elif score_name == 'elastic':
inits, inputs, nodes, outputs = _loss_elastic(existing_names, elem,
shape, output_name,
label_name, weight_name,
loss_name, **kwargs)
elif score_name == 'log':
shape = (None, 1)
inits, inputs, nodes, outputs = _loss_log(existing_names, elem, shape,
output_name, label_name,
weight_name, loss_name,
**kwargs)
else:
raise NotImplementedError( # pragma: no cover
f"Unexpected {score_name!r} value for score_name.")
if penalty is not None:
final_name = nodes[-1].output[0]
loss_name = _unique_name(existing_names, "loss_diff")
nodes[-1].output[0] = loss_name
names = []
for k, v in penalty.items():
if isinstance(v, float):
v = {'l2': v}
inits_to_add, nodes_to_add = penalty_loss_onnx(
k,
dtype=TENSOR_TYPE_TO_NP_TYPE[elem],
existing_names=existing_names,
**v)
names.append(nodes_to_add[-1].output[0])
nodes.extend(nodes_to_add)
inits.extend(inits_to_add)
# Operator Sum does not have a gradient.
if len(names) == 1:
pen_name = names[0]
else:
current = names[0]
for i in range(1, len(names)):
new_name = _unique_name(existing_names, "sumop")
nodes.append(make_node('Add', [current, names[i]], [new_name]))
current = new_name
pen_name = current
cst_shape = _unique_name(existing_names, "shapevect")
inits.append(
from_array(numpy.array([-1, 1], dtype=numpy.int64),
name=cst_shape))
loss_reshape = _unique_name(existing_names, "loss_reshape")
pen_reshape = _unique_name(existing_names, "penalty_reshape")
nodes.extend([
make_node("Reshape", [pen_name, cst_shape], [pen_reshape]),
make_node("Reshape", [loss_name, cst_shape], [loss_reshape])
])
nodes.append(
make_node('Add', [pen_reshape, loss_reshape], [final_name]))
inits = list(onx.graph.initializer) + inits
graph = make_graph(
list(onx.graph.node) + nodes, onx.graph.name,
list(onx.graph.input) + inputs,
outputs + [onx.graph.output[output_index]], inits)
onnx_model = make_model(graph)
onnx_model.ir_version = onx.ir_version
onnx_model.producer_name = onx.producer_name
onnx_model.producer_version = onx.producer_version
onnx_model.domain = onx.domain
onnx_model.model_version = onx.model_version
onnx_model.doc_string = onx.doc_string
if len(onx.metadata_props) > 0:
values = {p.key: p.value for p in onx.metadata_props}
set_model_props(onnx_model, values)
# fix opset import
del onnx_model.opset_import[:] # pylint: disable=E1101
for oimp in onx.opset_import:
op_set = onnx_model.opset_import.add() # pylint: disable=E1101
op_set.domain = oimp.domain
op_set.version = oimp.version
return _rewrite_op_no_grad(onnx_remove_node_unused(onnx_model))