def __call__(self, in_obj): in_axes = in_obj.axes if in_axes.channel_axis() is None: red_axes = ng.make_axes(in_axes.recurrent_axis()) + in_axes.batch_axes() else: red_axes = in_axes - in_axes.channel_axis() out_axes = in_axes - red_axes in_obj = ng.flatten(in_obj, out_axes | red_axes.flatten(force=True)) if self.gamma is None: self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes, initial_value=1.0) self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes, initial_value=0.0) self.gamma = ng.variable(axes=out_axes, initial_value=self.init_gamma, scope=self.scope).named('gamma') self.beta = ng.variable(axes=out_axes, initial_value=self.init_beta, scope=self.scope).named('beta') xmean = ng.mean(in_obj, out_axes=out_axes) xvar = ng.variance(in_obj, out_axes=out_axes) if Layer.inference_mode: return ng.unflatten(self.gamma * ((in_obj - self.gmean) * ng.reciprocal(ng.sqrt(self.gvar + self.eps))) + self.beta) else: return ng.sequential([ ng.assign(self.gmean, self.gmean * self.rho + xmean * (1.0 - self.rho)), ng.assign(self.gvar, self.gvar * self.rho + xvar * (1.0 - self.rho)), ng.unflatten(self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta) ])
def test_variance_sqrt_inverse(transformer_factory, input_tensor): inputs = input_tensor targets = ng.placeholder(inputs.axes) epsilon = 1e-3 inp_stat = ng.reciprocal( ng.sqrt( ng.variance(inputs, reduction_axes=inputs.axes.batch_axes()) + epsilon)) err = ng.sum(inp_stat - targets, out_axes=()) d_inputs = ng.deriv(err, inputs) with executor([err, d_inputs], inputs, targets) as comp_func: input_value = rng.uniform(-1, 1, inputs.axes) target_value = rng.uniform(-1, 1, targets.axes) ng_f_res, ng_b_res = comp_func(input_value, target_value) npv = np.var(input_value, axis=1, keepdims=True) + epsilon np_f_res = 1.0 / np.sqrt(npv) npv_delta = 2 * (input_value - np.mean(input_value, axis=1, keepdims=True)) np_b_res = -0.5 * np_f_res / npv * npv_delta np_f_res = np.sum(np_f_res - target_value) ng.testing.assert_allclose(np_f_res, ng_f_res, atol=1e-4, rtol=1e-4) ng.testing.assert_allclose(np_b_res, ng_b_res, atol=1e-4, rtol=1e-4)
def BatchNormalization(onnx_node, ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op x, scale, bias, mean, var = ng_inputs is_test = onnx_node.get_attribute_value('is_test', 1) spatial = onnx_node.get_attribute_value('spatial', 1) epsilon = onnx_node.get_attribute_value('epsilon', 1e-3) # @TODO: Implement learning mode support # momentum = onnx_node.get_attribute_value('momentum', 0.99) if not is_test: raise NotImplementedError( 'BatchNormalization node (%s): only `is_test` mode is currently ' 'supported.', onnx_node.name) if not spatial: raise NotImplementedError( 'BatchNormalization node (%s): only `spatial` mode is currently ' 'supported.', onnx_node.name) if len(x.axes) == 5: x = rename_axes(x, 'NCHWD') else: x = rename_axes(x, 'NCHW') mean = rename_axes(mean, 'C') scale = rename_axes(scale, 'C') bias = rename_axes(bias, 'C') var = rename_axes(var, 'C') ng_op = ng.unflatten(scale * ((x - mean) * ng.reciprocal(ng.sqrt(var + epsilon))) + bias) return cast_to_pos_axes(ng_op)
def test_reciprocal_derivative(transformer_factory, input_tensor): """TODO.""" p_u = input_tensor delta = .001 u = rng.uniform(.1, 5.0, p_u.axes) rec_u = ng.reciprocal(p_u) check_derivative(rec_u, p_u, delta, u, atol=1e-2, rtol=1e-2)
def test_reciprocal(transformer_factory, input_tensor): """TODO.""" p_u = input_tensor u = rng.uniform(.1, 5.0, p_u.axes) rec_u_np = np.reciprocal(u) rec_u = ng.reciprocal(p_u) with ExecutorFactory() as ex: rec_u_graph = ex.executor(rec_u, p_u)(u) ng.testing.assert_allclose(rec_u_np, rec_u_graph)
def Reciprocal(self, cntk_op, inputs): """ Returns element-wise reciprocal of inputs[0]. Arguments: inputs: List of inputs to this node. Returns: A ngraph Op. """ return ng.reciprocal(inputs[0]).named(cntk_op.uid)
def Reciprocal(self, cntk_op, inputs): """ Returns element-wise reciprocal of inputs[0]. Arguments: cntk_op: CNTK operation to be imported. inputs: List of inputs to this node. Returns: A ngraph Op. """ return ng.reciprocal(inputs[0]).named(cntk_op.uid)
def test_4d_chained(transformer_factory, input_axes): x_val = np.absolute(np.random.randn(*input_axes.lengths)) y_val = np.absolute(np.random.randn(*input_axes.lengths)) x = ng.constant(x_val, input_axes) y = ng.constant(y_val, input_axes) im = ng.reciprocal(x) out = ng.sum(ng.add(im, y), reduction_axes=input_axes[0]) with executor(out) as ex: graph_val = ex() np_val = np.sum(np.add(np.reciprocal(x_val), y_val), 0) np.testing.assert_allclose(graph_val, np_val, rtol=1e-4)
def test_4d_chained(transformer_factory, input_axes): # Limiting maximum absolute value for tensors elements to 7.9. # See description in function test_exit_condition above # Limitting minimum absolute value for tensors being input to reciprocal operation to 1/7.9 # # This is consequence of the above and flexpoint accuracy. # Numbers very small have poor absolute accuracy. When reciprocal of them is calculated the # results becomes very large and has even worse accuracy. When small numbers would be accepted # as an input to reciprocal in the test the absolute maximum value of the result is undefined # and so absolute tolerance. # To have possibility to set atol in the test and test could pass with it minimum element of # the tensor that is input to reciprocal operation has to be limited. is_flex = is_flex_factory(transformer_factory) clip_val_max = 7.9 if is_flex else 0 clip_val_min = 1.0 / 7.9 if is_flex else 0 x_val = rng.randn_abs_clip(input_axes, clip_min=clip_val_min, clip_max=clip_val_max) y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val_max) x = ng.constant(x_val, input_axes) y = ng.constant(y_val, input_axes) im = ng.reciprocal(x) out = ng.sum(ng.add(im, y), reduction_axes=input_axes[0]) with executor(out) as ex: graph_val = ex() np_val = np.sum(np.add(np.reciprocal(x_val), y_val), 0) # atol_multiplier = 15 * x_val.shape[0] # # x_val.shape[0] is number elements added together in operation # ng.sum(X, reduction_axes=input_axes[0]) # # 15 is calculated the following way: # # Input tensor has values from the range 1/7.9 - 7.9 # For DEC=12 absolute error is equal to 0.5*2^-12 = 0.000122 # 1/7.9 = 0.126582 with this error becomes 0.126704 # Reciprocal of 1/7.9 is 7.9 # Reciprocal of 1/7.9 + err = 7.892389 # Absolute difference is 0.007611 # It is 15.2 times larger then atol limit 5e-4 from Argon transformer ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4, atol_multiplier=15 * x_val.shape[0])
def test_reciprocal_derivative(transformer_factory): """TODO.""" N = ng.make_axis(name='N') W = ng.make_axis(name='W') delta = .001 W.length = 20 N.length = 128 axes = ng.make_axes([W, N]) p_u = ng.placeholder(axes) u = rng.uniform(.1, 5.0, p_u.axes) rec_u = ng.reciprocal(p_u) check_derivative(rec_u, p_u, delta, u, atol=1e-2, rtol=1e-2)
def construct_batchnorm_fprop_pattern(self): """ Generate graph op that represents a pattern for batchnorm fprop operation. self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta Returns: Single pattern that matches batchnorm fprop op """ self.batchnorm_fprop_input_tensor_label = "in_obj" self.batchnorm_fprop_gamma_label = "gamma" self.batchnorm_fprop_beta_label = "beta" self.batchnorm_fprop_variance_label = "variance" self.batchnorm_fprop_epsilon_label = "epsilon" self.batchnorm_fprop_mean_label = "mean" # bind the label to the op's which needed to be updated in the dict in_obj = PatternLabelOp(self.batchnorm_fprop_input_tensor_label, (lambda op: isinstance(op, ContiguousOp))) flatten_tensor = PatternSkipOp(in_obj, (lambda op: isinstance(op, Flatten))) gamma = PatternLabelOp(self.batchnorm_fprop_gamma_label, (lambda op: isinstance(op, BroadcastOp))) beta = PatternLabelOp(self.batchnorm_fprop_beta_label, (lambda op: isinstance(op, BroadcastOp))) variance = PatternLabelOp(self.batchnorm_fprop_variance_label, (lambda op: isinstance(op, Divide))) epsilon = PatternLabelOp(self.batchnorm_fprop_epsilon_label, (lambda op: isinstance(op, BroadcastOp))) mean = PatternLabelOp(self.batchnorm_fprop_mean_label, (lambda op: isinstance(op, Divide))) # construct the fprop batchnorm pattern matching the computation graph # ng.sqrt(xvar + self.eps) SqrtofVarianceAndEps = ng.sqrt(ng.add(variance, epsilon)) # ng.reciprocal(ng.sqrt(xvar + self.eps)) reciprocal_op = ng.reciprocal(SqrtofVarianceAndEps) reciprocal_op_w_braodcast = ng.PatternSkipOp(reciprocal_op, lambda op: isinstance(op, BroadcastOp)) mean_bcast = ng.PatternSkipOp(mean, lambda op: isinstance(op, BroadcastOp)) # (in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps)) mul_op_1 = ng.multiply(ng.subtract(flatten_tensor, mean_bcast), reciprocal_op_w_braodcast) # "self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) MultiplyGamma = ng.multiply(mul_op_1, gamma) # self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta AddBeta = ng.Unflatten(ng.Add(MultiplyGamma, beta)) return AddBeta
def test_reciprocal(transformer_factory): """TODO.""" N = ng.make_axis(name='N') W = ng.make_axis(name='W') W.length = 20 N.length = 128 axes = ng.make_axes([W, N]) p_u = ng.placeholder(axes) u = rng.uniform(.1, 5.0, p_u.axes) rec_u_np = np.reciprocal(u) rec_u = ng.reciprocal(p_u) ex = ExecutorFactory() rec_u_graph = ex.executor(rec_u, p_u)(u) np.testing.assert_allclose(rec_u_np, rec_u_graph)
def __call__(self, in_obj, channel_axes="C", spatial_axes=("D", "H", "W"), **kwargs): """ Arguments: in_obj (Op): Input op channel_axes (str): name of the expected channel axis type - defaults to "C" spatial_axes (tuple): names of expected depth, height and width axis types - defaults to "D", "H", and "W" """ if isinstance(spatial_axes, dict): spatial_axes = tuple( spatial_axes.get(name, name) for name in ("D", "H", "W")) elif isinstance(spatial_axes, tuple): if len(spatial_axes) < 3: raise ValueError( "spatial_axes must have length 3 (e.g. ('D', 'H', 'W'))") spatial_axes = tuple( name if name else default for name, default in zip(spatial_axes, ("D", "H", "W"))) orig_axes = in_obj.axes in_obj = reorder_spatial_axes(in_obj, channel_axes, spatial_axes) channel_axes = in_obj.axes.get_by_names(channel_axes) spatial_axes = in_obj.axes.get_by_names(*spatial_axes) filter_axes = self._filter_axes(channel_axes, spatial_axes) # mark 'K' as a shadow axis for the initializers. axes_map = shadow_axes_map(filter_axes.find_by_name('K')) filter_axes = ng.make_axes([ axis if axis.name != 'K' else list(axes_map.keys())[0] for axis in filter_axes ]) if not self.initialized: if not self.weight_norm: self.W = ng.variable(axes=filter_axes, initial_value=self.init, metadata={ "label": LABELS["weight"] }).named("W") else: self.v = ng.variable(axes=filter_axes, initial_value=self.init, metadata={ "label": LABELS["weight"] }).named("v") out_axes = ng.make_axes( [filter_axes.get_by_names("K__NG_SHADOW")]) v_norm = ng.mean(ng.square(self.v), out_axes=out_axes) self.g = ng.variable(axes=out_axes, initial_value=self.init, metadata={ "label": LABELS["weight"] }).named("g") self.W = self.g * self.v * ng.reciprocal( ng.sqrt(v_norm + 1e-3)) else: if filter_axes != self.W.axes: raise ValueError( ("{layer_name} layer has already been initialized with an " "input object which has resulted in filter axes: " "{existing_filter_axes}. This new input object has axes: " "{input_axes}, which implies the need for filter axes: " "{new_filter_axes} which are different than the existing " "filter axes.").format( layer_name=self.name, existing_filter_axes=self.W.axes, input_axes=in_obj.axes, new_filter_axes=filter_axes, )) output = ng.map_roles( self._conv_op(in_obj, channel_axes, spatial_axes), axes_map) # Reorder the output to match the input order output_axis_order = ng.make_axes( [output.axes.find_by_name(ax.name)[0] for ax in orig_axes]) # Remove introduced axes. If their length is > 1, then perhaps they should be kept slices = [ 0 if (ax not in orig_axes) and ax.length == 1 else slice(None) for ax in output.axes ] output = ng.tensor_slice(output, slices) # New axes with length > 1 may have been introduced. Add them to the end. output_axis_order = output_axis_order | output.axes return ng.axes_with_order(output, output_axis_order)
def Reciprocal(onnx_node, ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op return ng.reciprocal(ng_inputs[0])