def forward(self, input): out = self.smooth + FFx.sigmoid(self.gamma.data()) out = FF.reciprocal(out) out = input * out out = FFx.sigmoid(out) return out
def test_reciprocal(): inp = np.ones((2, INT_OVERFLOW)) inp[-1, -1] = 3 inp.attach_grad() with mx.autograd.record(): out = np.reciprocal(inp) out.backward() assert out.shape == inp.shape assert_almost_equal(out[-1, -1], np.array([1.0/3]), rtol=1e-5, atol=1e-5) assert inp.grad.shape == inp.shape assert_almost_equal(inp.grad[-1, -1], np.array([-1.0/3**2]), \ rtol=1e-5, atol=1e-5)
def tnmt_base(self, preds, labels): tpl = self.inner_prod(preds, labels) tpp = self.inner_prod(preds, preds) tll = self.inner_prod(labels, labels) num = tpl + self.smooth denum = 0.0 for d in range(self.depth): a = 2.**d b = -(2. * a - 1.) denum = denum + FF.reciprocal(a * (tpp + tll) + b * tpl + self.smooth) return num * denum * self.scale
def forward(self, data): var = np.power(data, 2).mean(-1, keepdims=True) data = data * np.reciprocal(np.sqrt(var + self._epsilon)) return data * self.gamma.data() + self.beta.data()
def forward(self, x): var = np.power(x.astype('float32'), 2).mean(-1, keepdims=True) x = x * np.reciprocal(np.sqrt(var + self.variance_epsilon)) if self.gemma.dtype == 'float16': x = x.astype('float16') return self.gemma * x