def forward(self, input):
out = self.smooth + FFx.sigmoid(self.gamma.data())
out = FF.reciprocal(out)
out = input * out
out = FFx.sigmoid(out)
return out
def test_reciprocal():
inp = np.ones((2, INT_OVERFLOW))
inp[-1, -1] = 3
inp.attach_grad()
with mx.autograd.record():
out = np.reciprocal(inp)
out.backward()
assert out.shape == inp.shape
assert_almost_equal(out[-1, -1], np.array([1.0/3]), rtol=1e-5, atol=1e-5)
assert inp.grad.shape == inp.shape
assert_almost_equal(inp.grad[-1, -1], np.array([-1.0/3**2]), \
rtol=1e-5, atol=1e-5)
def tnmt_base(self, preds, labels):
tpl = self.inner_prod(preds, labels)
tpp = self.inner_prod(preds, preds)
tll = self.inner_prod(labels, labels)
num = tpl + self.smooth
denum = 0.0
for d in range(self.depth):
a = 2.**d
b = -(2. * a - 1.)
denum = denum + FF.reciprocal(a *
(tpp + tll) + b * tpl + self.smooth)
return num * denum * self.scale
def forward(self, data):
var = np.power(data, 2).mean(-1, keepdims=True)
data = data * np.reciprocal(np.sqrt(var + self._epsilon))
return data * self.gamma.data() + self.beta.data()
def forward(self, x):
var = np.power(x.astype('float32'), 2).mean(-1, keepdims=True)
x = x * np.reciprocal(np.sqrt(var + self.variance_epsilon))
if self.gemma.dtype == 'float16':
x = x.astype('float16')
return self.gemma * x