def Selu(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Apply the scaled exponential linear unit function to the input tensor elementwise.
f(x) = gamma * (alpha * exp(x) - alpha) for x <= 0, f(x) = gamma * x for x > 0
"""
x = ng_inputs[0]
alpha = onnx_node.get_attribute_value('alpha', 1.6732)
gamma = onnx_node.get_attribute_value('gamma', 1.0507)
return (gamma * (ng.maximum(x, 0) + alpha * (ng.exp(ng.negative(ng.maximum(ng.negative(x), 0))) - 1)))
def Elu(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Apply the exponential linear unit function to the input tensor elementwise.
f(x) = alpha * (exp(x) - 1.) for x < 0, f(x) = x for x >= 0
"""
x = ng_inputs[0]
alpha = onnx_node.get_attribute_value('alpha', 1)
if not alpha < 0:
logger.warning('Elu node (%s): alpha value should be positive, but is: %s',
onnx_node.name, alpha)
return (ng.maximum(x, 0) + alpha * (ng.exp(ng.negative(ng.maximum(ng.negative(x), 0))) - 1))
def unary_op(op_str, a):
if op_str == "Abs":
return ng.abs(a)
elif op_str == "Acos":
return ng.acos(a)
elif op_str == "Asin":
return ng.asin(a)
elif op_str == "Atan":
return ng.atan(a)
elif op_str == "Ceiling":
return ng.ceiling(a)
elif op_str == "Cos":
return ng.cos(a)
elif op_str == "Cosh":
return ng.cosh(a)
elif op_str == "Floor":
return ng.floor(a)
elif op_str == "log":
return ng.log(a)
elif op_str == "exp":
return ng.exp(a)
elif op_str == "negative":
return ng.negative(a)
elif op_str == "Sign":
return ng.sign(a)
elif op_str == "Sin":
return ng.sin(a)
elif op_str == "Sinh":
return ng.sinh(a)
elif op_str == "Sqrt":
return ng.sqrt(a)
elif op_str == "Tan":
return ng.tan(a)
elif op_str == "Tanh":
return ng.tanh(a)
def Negate(self, cntk_op, inputs):
"""
Returns element-wise negation of inputs[0].
Arguments:
cntk_op: CNTK operation to be imported.
inputs: List of inputs to this node.
Returns:
A ngraph Op.
"""
assert len(inputs) == 1
return ng.negative(inputs[0]).named(cntk_op.uid)
def Neg(self, tf_node, inputs):
"""
Numerical negative value element-wise.
Arguments:
tf_node: NodeDef object, the tensorflow node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
x, name
"""
return ng.negative(inputs[0]).named(tf_node.name)
def Negative(self, c2_op, inputs):
"""
Numerical negative value element-wise.
Arguments:
c2_op: NodeDef object, the caffe2 node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the caffe2 node.
Inputs to c2_op:
x, name
"""
assert 1 == len(inputs)
return ng.negative(inputs[0]).named(c2_op.name)
def unary_op(op_str, a):
if op_str == 'Abs':
return ng.abs(a)
elif op_str == 'Acos':
return ng.acos(a)
elif op_str == 'Asin':
return ng.asin(a)
elif op_str == 'Atan':
return ng.atan(a)
elif op_str == 'Ceiling':
return ng.ceiling(a)
elif op_str == 'Cos':
return ng.cos(a)
elif op_str == 'Cosh':
return ng.cosh(a)
elif op_str == 'Floor':
return ng.floor(a)
elif op_str == 'log':
return ng.log(a)
elif op_str == 'exp':
return ng.exp(a)
elif op_str == 'negative':
return ng.negative(a)
elif op_str == 'Reverse':
return ng.reverse(a, np.array([1]), 'index')
elif op_str == 'Sign':
return ng.sign(a)
elif op_str == 'Sin':
return ng.sin(a)
elif op_str == 'Sinh':
return ng.sinh(a)
elif op_str == 'Sqrt':
return ng.sqrt(a)
elif op_str == 'Tan':
return ng.tan(a)
elif op_str == 'Tanh':
return ng.tanh(a)
def Neg(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Apply f(x) = -x to the input tensor elementwise (each element has flipped sign)."""
return ng.negative(ng_inputs[0])
def Sigmoid(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Apply the sigmoid function, f(x) = 1 / (1 + exp(-x)) to the input tensor elementwise."""
return 1 / (1 + ng.exp(ng.negative(ng_inputs[0])))
def construct_batchnorm_bprop_pattern(self):
"""
Generate graph op that represents a pattern for batchnorm backprop operation.
dgamma = np.sum(delta * xhat)
dbeta = np.sum(delta)
dx = gamma_scale * (delta - (xhat * dgamma + dbeta) / m)
In this pattern we are only generating the pattern for dx.
Returns:
Single pattern that matches batchnorm bprop op
"""
self.batchnorm_bprop_input_tensor = "input_tensor"
self.batchnorm_bprop_delta = "delta"
self.batchnorm_bprop_gamma_label = "gamma"
self.batchnorm_bprop_var_label = "var"
self.batchnorm_bprop_ivar_label = "ivar"
self.batchnorm_bprop_xmu1_label = "xmu1"
self.batchnorm_bprop_xmu2_label = "xmu2"
self.batchnorm_bprop_negative_inverse_sqrtvar = "negative_inverse_sqrtvar"
self.batchnorm_bprop_inverse_sqrtvar = "inverse_sqrtvar"
self.batchnorm_bprop_sqrtvar_label = "sqrtvar"
self.batchnorm_bprop_sqrsum = "sqrsum"
self.batchnorm_bprop_mean_1 = "mean_1"
self.batchnorm_bprop_mean_2 = "mean_2"
self.batchnorm_bprop_input_sum = "input_sum"
# bind the op's to the label
input_tensor = PatternLabelOp(self.batchnorm_bprop_input_tensor,
(lambda op: isinstance(op, Flatten)))
var = PatternLabelOp(self.batchnorm_bprop_var_label,
(lambda op: isinstance(op, Divide)))
gamma = PatternLabelOp(self.batchnorm_bprop_gamma_label,
(lambda op: isinstance(op, BroadcastOp)))
delta = PatternLabelOp(self.batchnorm_bprop_delta,
(lambda op: isinstance(op, Flatten)))
xmu1 = PatternLabelOp(self.batchnorm_bprop_xmu1_label,
(lambda op: isinstance(op, Subtract)))
xmu2 = PatternLabelOp(self.batchnorm_bprop_xmu2_label,
(lambda op: isinstance(op, Subtract)))
ivar = PatternLabelOp(self.batchnorm_bprop_ivar_label,
(lambda op: isinstance(op, BroadcastOp)))
negative_inverse_sqrtvar = PatternLabelOp(
self.batchnorm_bprop_negative_inverse_sqrtvar,
(lambda op: isinstance(op, NegativeOp)))
inverse_sqrtvar = PatternLabelOp(
self.batchnorm_bprop_inverse_sqrtvar,
(lambda op: isinstance(op, ReciprocalOp)))
sqrtvar = PatternLabelOp(self.batchnorm_bprop_sqrtvar_label,
(lambda op: isinstance(op, SqrtOp)))
sqrsum = PatternLabelOp(self.batchnorm_bprop_sqrsum,
(lambda op: isinstance(op, Sum)))
mean_1 = PatternLabelOp(self.batchnorm_bprop_mean_1,
(lambda op: isinstance(op, Divide)))
mean_2 = PatternLabelOp(self.batchnorm_bprop_mean_2,
(lambda op: isinstance(op, Divide)))
input_sum = PatternLabelOp(self.batchnorm_bprop_input_sum,
(lambda op: isinstance(op, Sum)))
constant_point_5 = ng.constant(0.5)
constant_point_5_w_broadcast = ng.PatternSkipOp(
constant_point_5, lambda op: isinstance(op, BroadcastOp))
constant_two = ng.constant(2)
constant_two_w_broadcast = ng.PatternSkipOp(
constant_two, lambda op: isinstance(op, BroadcastOp))
# construct the pattern
dxhat = Multiply(gamma, delta)
# divar = np.sum(dxhat*xmu, axis=0)
divar = Sum(Multiply(dxhat, xmu1))
# dxmu1 = dxhat * ivar
dxmu1 = Multiply(dxhat, ivar)
# dsqrtvar = -1. /(sqrtvar**2) * divar
dsqrtvar = Multiply(
Multiply(inverse_sqrtvar, negative_inverse_sqrtvar), divar)
# dvar = 0.5 * 1. /np.sqrt(var+eps) * dsqrtvar
dvar = Divide(Multiply(dsqrtvar, constant_point_5_w_broadcast),
sqrtvar)
# dsq = 1. / N * np.ones((N, D)) * dvar
dsq = Divide(Multiply(dvar, var), sqrsum)
dsq_w_broadcast = ng.PatternSkipOp(
dsq, (lambda op: isinstance(op, BroadcastOp)))
# dxmu2 = 2 * xmu * dsq
dxmu2 = Multiply(xmu2,
Multiply(constant_two_w_broadcast, dsq_w_broadcast))
# dx1 = (dxmu1 + dxmu2)
# dmu = -1 * np.sum(dxmu1 + dxmu2, axis=0)
# dx2 = 1. /N * np.ones((N,D)) * dmu
# dx = dx1 + dx2
dxmu2_mul = Multiply(Sum(ng.negative(dxmu2)), mean_2)
dxmu2_div = Divide(dxmu2_mul, input_sum)
dxmu2_div_w_broadcast = ng.PatternSkipOp(
dxmu2_div, (lambda op: isinstance(op, BroadcastOp)))
dxmu2_div_plus_dxmu2 = Add(dxmu2_div_w_broadcast, dxmu2)
dx1 = Add(dxmu1, dxmu2_div_plus_dxmu2)
dxmu1_mul = Multiply(Sum(ng.negative(dxmu1)), mean_1)
dxmu1_div = Divide(dxmu1_mul, Sum(input_tensor))
dxmu1_div_w_broadcast = ng.PatternSkipOp(
dxmu1_div, (lambda op: isinstance(op, BroadcastOp)))
dx = Add(dxmu1_div_w_broadcast, dx1)
return dx
def negative(x, name=None):
return ng.negative(x).named(name)
def test_dimshuffle_op(transformer_factory):
A = ng.make_axis().named('A')
B = ng.make_axis().named('B')
C = ng.make_axis().named('C')
D = ng.make_axis().named('D')
tests = [
{
'input_tensor': [
[
[
[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12],
],
[
[13, 14, 15, 16],
[17, 18, 19, 20],
[21, 22, 23, 24],
],
],
],
'input_tensor_axes': (A, B, C, D),
'output_tensor_axes': (B, D, A, C),
'axes_lengths': {
A: 1,
B: 2,
C: 3,
D: 4
},
'expected_result': [[
[[1, 5, 9]],
[[2, 6, 10]],
[[3, 7, 11]],
[[4, 8, 12]],
], [[[13, 17, 21]], [[14, 18, 22]], [[15, 19, 23]], [[16, 20,
24]]]]
},
{
'input_tensor': [[
[
[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12],
],
[
[13, 14, 15, 16],
[17, 18, 19, 20],
[21, 22, 23, 24],
],
],
[[
[25, 26, 27, 28],
[29, 30, 31, 32],
[33, 34, 35, 36],
],
[
[37, 38, 39, 40],
[41, 42, 43, 44],
[45, 46, 47, 48],
]]],
'input_tensor_axes': (A, B, C, D),
'output_tensor_axes': (B, D, A, C),
'axes_lengths': {
A: 2,
B: 2,
C: 3,
D: 4
},
'expected_result': [[[
[1, 5, 9],
[25, 29, 33],
], [
[2, 6, 10],
[26, 30, 34],
], [
[3, 7, 11],
[27, 31, 35],
], [
[4, 8, 12],
[28, 32, 36],
]],
[[
[13, 17, 21],
[37, 41, 45],
], [
[14, 18, 22],
[38, 42, 46],
], [[15, 19, 23], [39, 43, 47]],
[
[16, 20, 24],
[40, 44, 48],
]]]
},
]
for test in tests:
for axis, length in test['axes_lengths'].items():
axis.length = length
input_tensor = ng.placeholder(test['input_tensor_axes'])
input_tensor_value = np.array(test['input_tensor'], dtype=np.float32)
# This list of operations should add a dimshuffle operation to the graph.
a = ng.negative(input_tensor)
b = ng.axes_with_order(a, test['output_tensor_axes'])
c = ng.negative(b)
with executor(c, input_tensor) as ex:
out = ex(input_tensor_value)
ng.testing.assert_allclose(out, test['expected_result'])
def Neg(onnx_node, ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op
return ng.negative(ng_inputs[0])
