def __call__(self, x): """ Returns the Clipped Rectified Linear activation Arguments: x (Tensor or optree): Input value Returns: Tensor or optree: output activation """ return ng.minimum( ng.maximum(x, 0) + self.slope * ng.minimum(x, 0), self.cutoff)
def __call__(self, x): """ Returns the Exponential Linear activation Arguments: x (Tensor or optree): input value Returns: Tensor or optree: output activation """ return ng.maximum(x, 0) + self.alpha * (ng.exp(ng.minimum(x, 0)) - 1)
def __call__(self, x): """ Returns the Exponential Linear activation Arguments: x (Tensor or optree): Input value Returns: Tensor or optree: output activation """ return ng.maximum(x, 0) + self.slope * ng.minimum(0, x)
def Clip(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Limit input tensor values within specified interval.""" data = ng_inputs[0] data_elem_dtype = get_dtype(data.get_element_type()) max_value = onnx_node.get_attribute_value('max', np.finfo(data_elem_dtype).max) min_value = onnx_node.get_attribute_value('min', np.finfo(data_elem_dtype).min) return ng.minimum( ng.maximum(data, ng.constant(min_value, data_elem_dtype)), ng.constant(max_value, data_elem_dtype))
def clip_gradient_value(grad, clip_value=None): """ Element-wise clip a gradient tensor to between ``-clip_value`` and ``+clip_value``. Arguments: grad (Tensor): List of gradients for a single layer clip_value (float, optional): Value to element-wise clip gradients. Default: no clipping Returns: grad (list): List of clipped gradients. """ if clip_value is None: return grad else: return ng.minimum(ng.maximum(grad, -abs(clip_value)), abs(clip_value))
def HardSigmoid( onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Apply f(x) = max(0, min(1, alpha * x + beta)) function to tensor element-wise. :param onnx_node: The ONNX node representing this operation. :param ng_inputs: The input tensors. :return: The tensor with applied HardSigmoid operation. """ data = ng_inputs[0] data_type = get_dtype(data.get_element_type()).type alpha = onnx_node.get_attribute_value('alpha', float(0.2)) beta = onnx_node.get_attribute_value('beta', float(0.5)) return ng.maximum(data_type(0), ng.minimum(data_type(1), alpha * data + beta))
def test_clip(transformer_factory): H = ng.make_axis(length=5) W = ng.make_axis(length=4) axes = ng.make_axes([W, H]) p_x = ng.placeholder(axes) x = (2 * rng.uniform(0, 1, axes) - 1) * 20 clip_value = 10 clip_func = ng.minimum(ng.maximum(p_x, -abs(clip_value)), abs(clip_value)) # numpy results as expected results expected_result = np.clip(x, -abs(clip_value), abs(clip_value)) with ExecutorFactory() as ex: costfunc = ex.executor(clip_func, p_x) result = costfunc(x) ng.testing.assert_allclose(result, expected_result)
def clip_weight_value(weight, clip_value=None, min_value_override=None): """ Element-wise clip a weight tensor to between ``min_value_override`` and ``clip_value``. Arguments: weight (Tensor): List of gradients for a single layer clip_value (float, optional): Value to element-wise clip weights. Default: no clipping min_value (float, optional): Value to minimum value to element-wise clip weights. Default: -abs(clip_value) Returns: weight (list): List of clipped weights. """ if clip_value is None: return weight else: if min_value_override is None: min_value_override = -abs(clip_value) return ng.minimum(ng.maximum(weight, min_value_override), abs(clip_value))
def binary_op(op_str, a, b): if op_str == '+': return a + b elif op_str == 'Add': return ng.add(a, b) elif op_str == '-': return a - b elif op_str == 'Sub': return ng.subtract(a, b) elif op_str == '*': return a * b elif op_str == 'Mul': return ng.multiply(a, b) elif op_str == '/': return a / b elif op_str == 'Div': return ng.divide(a, b) elif op_str == 'Dot': return Dot(a, b) elif op_str == 'Equal': return ng.equal(a, b) elif op_str == 'Greater': return ng.greater(a, b) elif op_str == 'GreaterEq': return ng.greater_equal(a, b) elif op_str == 'Less': return ng.less(a, b) elif op_str == 'LessEq': return ng.less_equal(a, b) elif op_str == 'Maximum': return ng.maximum(a, b) elif op_str == 'Minimum': return ng.minimum(a, b) elif op_str == 'NotEqual': return ng.not_equal(a, b) elif op_str == 'Power': return ng.power(a, b)
def binary_op(op_str, a, b): if op_str == "+": return a + b elif op_str == "Add": return ng.add(a, b) elif op_str == "-": return a - b elif op_str == "Sub": return ng.subtract(a, b) elif op_str == "*": return a * b elif op_str == "Mul": return ng.multiply(a, b) elif op_str == "/": return a / b elif op_str == "Div": return ng.divide(a, b) elif op_str == "Equal": return ng.equal(a, b) elif op_str == "Greater": return ng.greater(a, b) elif op_str == "GreaterEq": return ng.greater_equal(a, b) elif op_str == "Less": return ng.less(a, b) elif op_str == "LessEq": return ng.less_equal(a, b) elif op_str == "Maximum": return ng.maximum(a, b) elif op_str == "Minimum": return ng.minimum(a, b) elif op_str == "NotEqual": return ng.not_equal(a, b) elif op_str == "Power": return ng.power(a, b)