def test_sqnorm_array_multi_gpu(self):
x0 = cuda.to_gpu(self.x, device=0)
x1 = cuda.to_gpu(self.x, device=1)
a0 = cuda.to_gpu(self.a, device=0)
a1 = cuda.to_gpu(self.a, device=1)
self.assertAlmostEqual(
gradient_clipping._sum_sqnorm([self.x, self.a, x0, a0, x1, a1]),
8.75 * 3)
def check_clipping(self, multiplier):
w = self.target.param.data
g = self.target.param.grad
xp = backend.get_array_module(w)
norm = xp.sqrt(gradient_clipping._sum_sqnorm(g))
threshold = norm * multiplier
if multiplier < 1:
expect = w - g * multiplier
else:
expect = w - g
opt = optimizers.SGD(lr=1)
opt.setup(self.target)
opt.add_hook(optimizer_hooks.GradientClipping(threshold))
opt.update()
testing.assert_allclose(expect, w)
def test_sqnorm_array(self):
x = cuda.to_gpu(self.x)
a = cuda.to_gpu(self.a)
self.assertAlmostEqual(
gradient_clipping._sum_sqnorm([self.x, self.a, x, a]), 8.75 * 2)
def test_sqnorm_scalar_gpu(self):
a = cuda.to_gpu(self.a)
self.assertAlmostEqual(gradient_clipping._sum_sqnorm([a]), 4)
def test_sqnorm_gpu(self):
x = cuda.to_gpu(self.x)
self.assertAlmostEqual(gradient_clipping._sum_sqnorm([x]), 4.75)
def test_sqnorm_scalar_cpu(self):
self.assertAlmostEqual(gradient_clipping._sum_sqnorm([self.a]), 4)
def test_sqnorm_cpu(self):
# \Sum_{n=0}^{5} (-1.0+0.5n)**2 = 4.75
self.assertAlmostEqual(gradient_clipping._sum_sqnorm([self.x]), 4.75)