def _backward_gpu(self, context, dy, **kwargs):
gw, gx, gdy, gm, gv = map(
get_gpu,
(self.attrs._w, self.attrs._x, dy, self.attrs._m, self.attrs._v))
dx, dw, db = (g.ones_like_me() for g in (gx, gw, gw))
ax = self.attrs._axs
with cu.cudnn_handler() as handle:
cu.cuBatchNormalizatoinBackward(handle,
gx,
gw,
gdy,
gm,
gv,
dx,
dw,
db,
mode=ax)
if isinstance(self.attrs._x, Node):
self.attrs._x._update_diff(context, dx, **kwargs)
if isinstance(self.attrs._w, Node):
self.attrs._w._update_diff(context, dw, **kwargs)
if isinstance(self.attrs._b, Node):
self.attrs._b._update_diff(context, db, **kwargs)
def _backward_gpu(self, context, dy, **kwargs):
dx = get_gpu(self.attrs._x).empty_like_me()
with cu.cudnn_handler() as handle:
cu.cuPoolingBackward(handle, self.attrs._pool_desc, self.attrs._x,
self, dy, dx)
if isinstance(self.attrs._x, Node):
self.attrs._x._update_diff(context, dx, **kwargs)
def _backward_gpu(self, context, dy, **kwargs):
if isinstance(self.attrs._x, Node):
dx = get_gpu(self).empty_like_me()
with cu.cudnn_handler() as handle:
cu.cuLocalResponseNormalizationBackward(
handle, self.attrs._lrn_desc, self.attrs._x, self, dx, dy)
self.attrs._x._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, x, n, k, a, b):
lrn_desc = cu.LRNDescriptor(n, a, b, k)
y = get_gpu(x).empty_like_me()
with cu.cudnn_handler() as handle:
cu.cuLocalResponseNormalizationForward(handle, lrn_desc, x, y)
ret = cls._create_node(y)
ret.attrs._x = x
ret.attrs._lrn_desc = lrn_desc
return ret
def _oper_gpu(cls, x, in_shape, out_shape, karnel, stride, padding):
N = x.shape[0]
pool_desc = cu.createPoolingDescriptor(karnel, padding, stride, pool_mode=1)
y = GPUValue(shape=tuple([N, ] + list(out_shape)))
with cu.cudnn_handler() as handle:
cu.cuPoolingForward(handle, pool_desc, x, y)
ret = cls._create_node(y)
ret.attrs._pool_desc = pool_desc
ret.attrs._x = x
return ret
def _backward_gpu(self, context, dy, **kwargs):
if isinstance(self.attrs._arg, Node):
with cu.cudnn_handler() as handle:
dx = get_gpu(self).empty_like_me()
cu.cuSoftmaxBackward(handle,
get_gpu(self),
get_gpu(dy),
dx,
mode=1)
self.attrs._arg._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, lhs, rhs):
N = lhs.shape[0]
z = get_gpu(lhs).empty_like_me()
tmp1 = get_gpu(lhs).empty_like_me()
with cu.cudnn_handler() as handle:
cu.cuSoftmaxForward(handle, lhs, z, mode=1)
cu.cucross_entropy(get_gpu(z), get_gpu(rhs), get_gpu(tmp1))
loss = -cu.cusum(get_gpu(tmp1)) / N
ret = cls._create_node(loss)
ret.attrs._z = z
ret.attrs._lhs = lhs
ret.attrs._rhs = rhs
return ret
def _backward_gpu(self, context, dy):
dw, db, dx = (get_gpu(g).empty_like_me()
for g in (self.attrs._w, self.attrs._b, self.attrs._x))
with cu.cudnn_handler() as handle:
cu.cuConvolutionBackward(handle, self.attrs._conv_desc, self.attrs._filter_desc,
self.attrs._x, self.attrs._w, dy, dw, db, dx)
if isinstance(self.attrs._w, Node):
self.attrs._w._update_diff(context, dw)
if isinstance(self.attrs._x, Node):
self.attrs._x._update_diff(context, dx)
if isinstance(self.attrs._b, Node):
self.attrs._b._update_diff(context, db)
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding):
conv_desc = cu.ConvolutionDescriptor(padding, stride, precision)
filter_desc = cu.FilterDescriptor(w.shape, precision)
N = x.shape[0]
# TODO: dirty code
z = GPUValue(shape=tuple([N, ] + list(out_shape)))
with cu.cudnn_handler() as handle:
cu.cuConvolutionBackwardData(handle, conv_desc, filter_desc, w, x, z)
if b is not None:
cu.cu_add_bias(get_gpu(b), z)
ret = cls._create_node(z)
ret.attrs._conv_desc = conv_desc
ret.attrs._filter_desc = filter_desc
ret.attrs._x = x
ret.attrs._w = w
ret.attrs._b = b
return ret
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding):
N = x.shape[0]
conv_desc = cu.createConvplutionDescriptor(padding, stride, precision)
filter_desc = cu.createFilterDescriptor(w.shape, precision)
# TODO: dirty code
y = GPUValue(shape=tuple([N, ] + list(out_shape)))
with cu.cudnn_handler() as handle:
cu.cuConvolutionForward(handle, conv_desc, filter_desc, x, w, y)
if b is not None:
cu.cuadd(get_gpu(y), get_gpu(b), get_gpu(y))
ret = cls._create_node(y)
ret.attrs._conv_desc = conv_desc
ret.attrs._filter_desc = filter_desc
ret.attrs._x = x
ret.attrs._w = w
ret.attrs._b = b
return ret
def _oper_gpu(cls, x, w, b, momentum, mov_m, mov_s, inference, mode,
epsilon):
if mode == BATCH_NORMALIZE_FEATUREMAP:
axs = 1
else:
axs = 0
y, mean, sq_var = (get_gpu(g).empty_like_me() for g in (x, w, w))
mov_m = get_gpu(mov_m)
mov_s = get_gpu(mov_s)
mv_m = mov_m if isinstance(mov_m,
GPUValue) else get_gpu(w).zeros_like_me()
mv_v = mov_s if isinstance(mov_s,
GPUValue) else get_gpu(w).zeros_like_me()
with cu.cudnn_handler() as handle:
cu.cuBatchNormalizatoinForward(handle,
x,
mv_m,
mv_v,
w,
b,
y,
mean,
sq_var,
momentum=momentum,
mode=axs,
inference=inference,
eps=epsilon)
ret = cls._create_node(y)
ret.attrs._axs = axs
ret.attrs._x = x
ret.attrs._w = w
ret.attrs._b = b
ret.attrs._m = mean
ret.attrs._v = sq_var
if not inference:
ret.attrs._mov_m = mv_m
ret.attrs._mov_v = mv_v
return ret
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding):
N = x.shape[0]
conv_desc = cu.ConvolutionDescriptor(padding, stride, precision)
filter_desc = cu.FilterDescriptor(w.shape, precision)
y = GPUValue(shape=tuple([
N,
] + list(out_shape)))
with cu.cudnn_handler() as handle:
cu.cuConvolutionForward(handle, conv_desc, filter_desc, x, w, y)
if b is not None:
cu.cu_add_bias(get_gpu(b), y)
# assert type(x) is not np.ndarray
ret = cls._create_node(y)
ret.attrs._conv_desc = conv_desc
ret.attrs._filter_desc = filter_desc
ret.attrs._x = x
ret.attrs._w = w
ret.attrs._b = b
return ret
def _oper_gpu(cls, arg):
z = get_gpu(arg).empty_like_me()
with cu.cudnn_handler() as handle:
cu.cuSoftmaxForward(handle, arg, z, mode=1)
return z
