def test_variance_sqrt_inverse(transformer_factory, input_tensor): inputs = input_tensor targets = ng.placeholder(inputs.axes) epsilon = 1e-3 inp_stat = ng.reciprocal( ng.sqrt( ng.variance(inputs, reduction_axes=inputs.axes.batch_axes()) + epsilon)) err = ng.sum(inp_stat - targets, out_axes=()) d_inputs = ng.deriv(err, inputs) with executor([err, d_inputs], inputs, targets) as comp_func: input_value = rng.uniform(-1, 1, inputs.axes) target_value = rng.uniform(-1, 1, targets.axes) ng_f_res, ng_b_res = comp_func(input_value, target_value) npv = np.var(input_value, axis=1, keepdims=True) + epsilon np_f_res = 1.0 / np.sqrt(npv) npv_delta = 2 * (input_value - np.mean(input_value, axis=1, keepdims=True)) np_b_res = -0.5 * np_f_res / npv * npv_delta np_f_res = np.sum(np_f_res - target_value) ng.testing.assert_allclose(np_f_res, ng_f_res, atol=1e-4, rtol=1e-4) ng.testing.assert_allclose(np_b_res, ng_b_res, atol=1e-4, rtol=1e-4)
def train_outputs(self, in_obj): in_axes = in_obj.axes.sample_axes() red_axes = ng.make_axes() if len(in_axes.role_axes(ar.features_input)) != 0: red_axes += in_axes.sample_axes() - in_axes.role_axes( ar.features_input) red_axes += in_obj.axes.batch_axes() out_axes = in_axes - red_axes self.gamma = self.gamma or ng.variable( axes=out_axes, initial_value=1.0).named('gamma') self.beta = self.beta or ng.variable(axes=out_axes, initial_value=0.0).named('beta') self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes, initial_value=1.0) self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes, initial_value=0.0) xmean = ng.mean(in_obj, reduction_axes=red_axes) xvar = ng.variance(in_obj, reduction_axes=red_axes) return ng.sequential([ ng.assign(self.gmean, self.gmean * self.rho + xmean * (1.0 - self.rho)), ng.assign(self.gvar, self.gvar * self.rho + xvar * (1.0 - self.rho)), self.gamma * (in_obj - xmean) / ng.sqrt(xvar + self.eps) + self.beta ])
def __call__(self, in_obj): in_axes = in_obj.axes if in_axes.channel_axis() is None: red_axes = ng.make_axes(in_axes.recurrent_axis()) + in_axes.batch_axes() else: red_axes = in_axes - in_axes.channel_axis() out_axes = in_axes - red_axes in_obj = ng.flatten(in_obj, out_axes | red_axes.flatten(force=True)) if self.gamma is None: self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes, initial_value=1.0) self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes, initial_value=0.0) self.gamma = ng.variable(axes=out_axes, initial_value=self.init_gamma, scope=self.scope).named('gamma') self.beta = ng.variable(axes=out_axes, initial_value=self.init_beta, scope=self.scope).named('beta') xmean = ng.mean(in_obj, out_axes=out_axes) xvar = ng.variance(in_obj, out_axes=out_axes) if Layer.inference_mode: return ng.unflatten(self.gamma * ((in_obj - self.gmean) * ng.reciprocal(ng.sqrt(self.gvar + self.eps))) + self.beta) else: return ng.sequential([ ng.assign(self.gmean, self.gmean * self.rho + xmean * (1.0 - self.rho)), ng.assign(self.gvar, self.gvar * self.rho + xvar * (1.0 - self.rho)), ng.unflatten(self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta) ])
def test_variance_wgrad(transformer_factory): ax = ng.name_scope('x') ax.N = ng.make_axis(128, batch=True) ax.Y = ng.make_axis(100) inputs = ng.placeholder([ax.Y, ax.N]) targets = ng.placeholder([ax.Y, ax.N]) inp_stat = ng.variance(inputs, reduction_axes=inputs.axes.batch_axes()) err = ng.sum(inp_stat - targets, out_axes=()) d_inputs = ng.deriv(err, inputs) comp_func = executor([err, d_inputs], inputs, targets) input_value = rng.uniform(-0.1, 0.1, inputs.axes) target_value = rng.uniform(-0.1, 0.1, targets.axes) ng_f_res, ng_b_res = comp_func(input_value, target_value) np_f_res = np.sum( np.var(input_value, axis=1, keepdims=True) - target_value) np.testing.assert_allclose(np_f_res, ng_f_res, atol=1e-4, rtol=1e-4) np_b_res = 2 * (input_value - np.mean(input_value, axis=1, keepdims=True)) np.testing.assert_allclose(np_b_res, ng_b_res, atol=1e-4, rtol=1e-4)
def test_variance_wgrad(transformer_factory, input_tensor): inputs = input_tensor targets = ng.placeholder(inputs.axes) inp_stat = ng.variance(inputs, reduction_axes=inputs.axes.batch_axes()) err = ng.sum(inp_stat - targets, out_axes=()) d_inputs = ng.deriv(err, inputs) with executor([err, d_inputs], inputs, targets) as comp_func: input_value = rng.uniform(-0.1, 0.1, inputs.axes) target_value = rng.uniform(-0.1, 0.1, targets.axes) ng_f_res, ng_b_res = comp_func(input_value, target_value) np_f_res = np.sum(np.var(input_value, axis=1, keepdims=True) - target_value) ng.testing.assert_allclose(np_f_res, ng_f_res, atol=1e-4, rtol=1e-4) np_b_res = 2 * (input_value - np.mean(input_value, axis=1, keepdims=True)) ng.testing.assert_allclose(np_b_res, ng_b_res, atol=1e-4, rtol=1e-4)