def __call__(self, x):
"""
Returns the Clipped Rectified Linear activation
Arguments:
x (Tensor or optree): Input value
Returns:
Tensor or optree: output activation
"""
return ng.minimum(
ng.maximum(x, 0) + self.slope * ng.minimum(x, 0), self.cutoff)
def __call__(self, x):
"""
Returns the Exponential Linear activation
Arguments:
x (Tensor or optree): input value
Returns:
Tensor or optree: output activation
"""
return ng.maximum(x, 0) + self.alpha * (ng.exp(ng.minimum(x, 0)) - 1)
def __call__(self, x):
"""
Returns the Exponential Linear activation
Arguments:
x (Tensor or optree): Input value
Returns:
Tensor or optree: output activation
"""
return ng.maximum(x, 0) + self.slope * ng.minimum(0, x)
def Clip(onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Limit input tensor values within specified interval."""
data = ng_inputs[0]
data_elem_dtype = get_dtype(data.get_element_type())
max_value = onnx_node.get_attribute_value('max',
np.finfo(data_elem_dtype).max)
min_value = onnx_node.get_attribute_value('min',
np.finfo(data_elem_dtype).min)
return ng.minimum(
ng.maximum(data, ng.constant(min_value, data_elem_dtype)),
ng.constant(max_value, data_elem_dtype))
def clip_gradient_value(grad, clip_value=None):
"""
Element-wise clip a gradient tensor to between ``-clip_value`` and ``+clip_value``.
Arguments:
grad (Tensor): List of gradients for a single layer
clip_value (float, optional): Value to element-wise clip gradients. Default: no clipping
Returns:
grad (list): List of clipped gradients.
"""
if clip_value is None:
return grad
else:
return ng.minimum(ng.maximum(grad, -abs(clip_value)), abs(clip_value))
def HardSigmoid(
onnx_node,
ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""Apply f(x) = max(0, min(1, alpha * x + beta)) function to tensor element-wise.
:param onnx_node: The ONNX node representing this operation.
:param ng_inputs: The input tensors.
:return: The tensor with applied HardSigmoid operation.
"""
data = ng_inputs[0]
data_type = get_dtype(data.get_element_type()).type
alpha = onnx_node.get_attribute_value('alpha', float(0.2))
beta = onnx_node.get_attribute_value('beta', float(0.5))
return ng.maximum(data_type(0),
ng.minimum(data_type(1), alpha * data + beta))
def test_clip(transformer_factory):
H = ng.make_axis(length=5)
W = ng.make_axis(length=4)
axes = ng.make_axes([W, H])
p_x = ng.placeholder(axes)
x = (2 * rng.uniform(0, 1, axes) - 1) * 20
clip_value = 10
clip_func = ng.minimum(ng.maximum(p_x, -abs(clip_value)), abs(clip_value))
# numpy results as expected results
expected_result = np.clip(x, -abs(clip_value), abs(clip_value))
with ExecutorFactory() as ex:
costfunc = ex.executor(clip_func, p_x)
result = costfunc(x)
ng.testing.assert_allclose(result, expected_result)
def clip_weight_value(weight, clip_value=None, min_value_override=None):
"""
Element-wise clip a weight tensor to between ``min_value_override`` and ``clip_value``.
Arguments:
weight (Tensor): List of gradients for a single layer
clip_value (float, optional): Value to element-wise clip weights. Default: no clipping
min_value (float, optional): Value to minimum value to element-wise clip
weights. Default: -abs(clip_value)
Returns:
weight (list): List of clipped weights.
"""
if clip_value is None:
return weight
else:
if min_value_override is None:
min_value_override = -abs(clip_value)
return ng.minimum(ng.maximum(weight, min_value_override),
abs(clip_value))
def binary_op(op_str, a, b):
if op_str == '+':
return a + b
elif op_str == 'Add':
return ng.add(a, b)
elif op_str == '-':
return a - b
elif op_str == 'Sub':
return ng.subtract(a, b)
elif op_str == '*':
return a * b
elif op_str == 'Mul':
return ng.multiply(a, b)
elif op_str == '/':
return a / b
elif op_str == 'Div':
return ng.divide(a, b)
elif op_str == 'Dot':
return Dot(a, b)
elif op_str == 'Equal':
return ng.equal(a, b)
elif op_str == 'Greater':
return ng.greater(a, b)
elif op_str == 'GreaterEq':
return ng.greater_equal(a, b)
elif op_str == 'Less':
return ng.less(a, b)
elif op_str == 'LessEq':
return ng.less_equal(a, b)
elif op_str == 'Maximum':
return ng.maximum(a, b)
elif op_str == 'Minimum':
return ng.minimum(a, b)
elif op_str == 'NotEqual':
return ng.not_equal(a, b)
elif op_str == 'Power':
return ng.power(a, b)
def binary_op(op_str, a, b):
if op_str == "+":
return a + b
elif op_str == "Add":
return ng.add(a, b)
elif op_str == "-":
return a - b
elif op_str == "Sub":
return ng.subtract(a, b)
elif op_str == "*":
return a * b
elif op_str == "Mul":
return ng.multiply(a, b)
elif op_str == "/":
return a / b
elif op_str == "Div":
return ng.divide(a, b)
elif op_str == "Equal":
return ng.equal(a, b)
elif op_str == "Greater":
return ng.greater(a, b)
elif op_str == "GreaterEq":
return ng.greater_equal(a, b)
elif op_str == "Less":
return ng.less(a, b)
elif op_str == "LessEq":
return ng.less_equal(a, b)
elif op_str == "Maximum":
return ng.maximum(a, b)
elif op_str == "Minimum":
return ng.minimum(a, b)
elif op_str == "NotEqual":
return ng.not_equal(a, b)
elif op_str == "Power":
return ng.power(a, b)
