def backward_gpu(self, inputs, grad_outputs):
gh = grad_outputs[0]
x, h_tm1 = inputs
N = x.shape[0]
gz = cuda.empty_like(gh)
if self.act_func_str in ('tanh', 'sigmoid'):
#backpropagate non-linearities
gz = self.cu_dact_func(gy=gh, y=self.h, out=gz)
# compute gradient with respect to the hidden input state
gh_tm1 = cuk.dot(gz, self.V, out=self.h)
elif self.act_func_str in ('leakyrelu', 'relu'):
#backpropagate non-linearities
gz = self.cu_dact_func(x=self.z, gy=gh, out=gz)
# compute gradient with respect to the hidden input state
gh_tm1 = cuk.dot(gz, self.V, out=self.z)
else:
raise NotImplementedError('the activation function is not available')
#backpropagate linear function
if self.hot:
gx = None
cuk.dothot(gz, x, in_size=self.in_size, out=self.gW)
else:
gx = cuda.empty_like(x)
cuk.dot(gz, self.W, out=gx)
cuk.dotAdd(gz, x, C=self.gW, transa='t')
cuk.dotAdd(gz, h_tm1, C=self.gV, transa='t')
if not self.nobias:
gb_ones = cuda.ones((1,N),dtype=np.float32)
cuk.dotAdd(gb_ones, gz, C=self.gb)
return gx, gh_tm1
def _partial_reduce(x):
global _one
out_axis, sum_axis = x.shape
one = _one
if one is None or one.size < sum_axis:
one = cuda.ones(sum_axis)
_one = one
one = one[:sum_axis]
handle = cuda.get_cublas_handle()
ret = cuda.empty(out_axis)
cuda.cublas.cublasSgemv(handle, 't', sum_axis, out_axis,
numpy.float32(1.0), x.gpudata, sum_axis, one.gpudata,
1, numpy.float32(0.0), ret.gpudata, 1)
return ret
def _partial_reduce(x):
global _one
out_axis, sum_axis = x.shape
one = _one
if one is None or one.size < sum_axis:
one = cuda.ones(sum_axis)
_one = one
one = one[:sum_axis]
handle = cuda.get_cublas_handle()
ret = cuda.empty(out_axis)
cuda.cublas.cublasSgemv(handle, 't', sum_axis, out_axis,
numpy.float32(1.0), x.gpudata, sum_axis,
one.gpudata, 1, numpy.float32(0.0), ret.gpudata, 1)
return ret
def backward_gpu(self, inputs, grad_outputs):
x, targets = inputs
gloss = grad_outputs[0]
N = x.shape[0]
coeff = gloss*1.0/N
cuda.culinalg.scale(coeff, self.diff, alpha_real=True)
gy = self.diff
gtargets = None
#backpropagate linear function
gx = cuda.empty_like(x)
cuk.dot(gy, self.W, out=gx)
cuk.dotAdd(gy, x, C=self.gW, transa='t')
if not self.nobias:
gb_ones = cuda.ones((1,N),dtype=np.float32)
cuk.dotAdd(gb_ones, gy, C=self.gb)
return gx, gtargets
def backward_gpu(self, inputs, grad_outputs):
x, targets = inputs
gloss = cuda.to_gpu(grad_outputs[0])
N = x.shape[0]
gtargets = None # the function is non-differentiable with respect to the targets
#backpropagate Softmax Cross Entropy Error
gz = self.probs
gz = cuk.dSoftmaxCrossEntropy(gz, targets, gloss)
#backpropagate linear function
gx = cuda.empty_like(x)
cuk.dot(gz, self.W, out=gx)
cuk.dotAdd(gz, x, C=self.gW, transa='t')
if not self.nobias:
gb_ones = cuda.ones((1,N),dtype=np.float32)
cuk.dotAdd(gb_ones, gz, C=self.gb)
return gx, gtargets
def backward_gpu(self, inputs, grad_outputs):
gh, gc = grad_outputs
x, h_tm1, c_tm1 = inputs
if gh is None:
gh = cuda.to_gpu(np.array([[0]], dtype=np.float32))
gh_is_none = 1
else:
gh_is_none = 0
if gc is None:
gc = cuda.to_gpu(np.array([[0]], dtype=np.float32))
gc_is_none = 1
else:
gc_is_none = 0
gc_tm1 = self.c
cuk.lstm_backward(c=self.c, z=self.z, gh=gh,
gc=gc, c_tm1=c_tm1,
gc_is_none=gc_is_none, gh_is_none=gh_is_none)
gz = self.z
gh_tm1 = cuk.dot(gz, self.U, out=self.h)
# compute gradient with respect to the input x
gx = cuda.empty_like(x)
gx = cuk.dot(gz, self.W, out=gx)
# compute gradients of weight matrices
cuk.dotAdd(gz, x, C=self.gW, transa='t')
cuk.dotAdd(gz, h_tm1, C=self.gU, transa='t')
if not self.nobias:
N = x.shape[0]
gb_ones = cuda.ones((1,N),dtype=np.float32)
cuk.dotAdd(gb_ones, gz, C=self.gb)
return gx, gh_tm1, gc_tm1
def backward_gpu(self, inputs, grad_outputs):
x, h_tm1 = inputs
gh = grad_outputs[0]
gu, gh_tilde, gh_tm1, gr, ghr = cuk.gru_backward(
gu=self.h, h_tm1=h_tm1, h_tilde=self.h_tilde,
gh_tilde=self.h_tilde, gh=gh, u=self.u,
gh_tm1=self.u, gr=self.HV, r=self.r,
HV=self.HV, ghr=self.r)
gx = cuda.empty_like(x)
cuk.dot(gu, self.Wu, out=gx)
cuk.dotAdd(gr, self.Wr, C=gx)
cuk.dotAdd(gh_tilde, self.Wh, C=gx)
cuk.dotAdd(ghr, self.Vh, C=gh_tm1)
cuk.dotAdd(gr, self.Vr, C=gh_tm1)
cuk.dotAdd(gu, self.Vu, C=gh_tm1)
cuk.dotAdd(gu, x, C=self.gWu, transa='t')
cuk.dotAdd(gu, h_tm1, C=self.gVu, transa='t')
cuk.dotAdd(gr, x, C=self.gWr, transa='t')
cuk.dotAdd(gr, h_tm1, C=self.gVr, transa='t')
cuk.dotAdd(gh_tilde, x, C=self.gWh, transa='t')
cuk.dotAdd(ghr, h_tm1, C=self.gVh, transa='t')
if not self.nobias:
N = x.shape[0]
gb_ones = cuda.ones((1,N),dtype=np.float32)
cuk.dotAdd(gb_ones, gu, C=self.gbu)
cuk.dotAdd(gb_ones, gr, C=self.gbr)
cuk.dotAdd(gb_ones, gh_tilde, C=self.gbh)
return gx, gh_tm1
def test_add_constant_allocation(self):
x = 0
y = Variable(cuda.ones((1,)))
z = y + x
self.assertEqual(1, z.data.get()[0])
self.assertTrue(hasattr(z.data.gpudata, 'device'))
def _ones(gpu, *shape):
if gpu:
return chainer.Variable(cuda.ones(shape).astype(numpy.float32))
return chainer.Variable(numpy.ones(shape).astype(numpy.float32))
def test_add_constant_allocation(self):
x = 0
y = chainer.Variable(cuda.ones((1, )))
z = y + x
self.assertEqual(1, z.data.get()[0])
self.assertTrue(hasattr(z.data.gpudata, 'device'))
def _ones(gpu, *shape):
if gpu:
return chainer.Variable(cuda.ones(shape).astype(numpy.float32))
return chainer.Variable(numpy.ones(shape).astype(numpy.float32))
def test_add_constant_allocation(self):
x = 0
y = chainer.Variable(cuda.ones((1, )))
z = y + x
self.assertEqual(1, z.data.get()[0])
def test_add_constant_allocation(self):
x = 0
y = chainer.Variable(cuda.ones((1,)))
z = y + x
self.assertEqual(1, z.data.get()[0])