def _backward_gpu(self, context, dy, **kwargs): gw, gx, gdy, gm, gv = map( get_gpu, (self.attrs._w, self.attrs._x, dy, self.attrs._m, self.attrs._v)) dx, dw, db = (g.ones_like_me() for g in (gx, gw, gw)) ax = self.attrs._axs with cu.cudnn_handler() as handle: cu.cuBatchNormalizatoinBackward(handle, gx, gw, gdy, gm, gv, dx, dw, db, mode=ax) if isinstance(self.attrs._x, Node): self.attrs._x._update_diff(context, dx, **kwargs) if isinstance(self.attrs._w, Node): self.attrs._w._update_diff(context, dw, **kwargs) if isinstance(self.attrs._b, Node): self.attrs._b._update_diff(context, db, **kwargs)
def _backward_gpu(self, context, dy, **kwargs): dx = get_gpu(self.attrs._x).empty_like_me() with cu.cudnn_handler() as handle: cu.cuPoolingBackward(handle, self.attrs._pool_desc, self.attrs._x, self, dy, dx) if isinstance(self.attrs._x, Node): self.attrs._x._update_diff(context, dx, **kwargs)
def _backward_gpu(self, context, dy, **kwargs): if isinstance(self.attrs._x, Node): dx = get_gpu(self).empty_like_me() with cu.cudnn_handler() as handle: cu.cuLocalResponseNormalizationBackward( handle, self.attrs._lrn_desc, self.attrs._x, self, dx, dy) self.attrs._x._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, x, n, k, a, b): lrn_desc = cu.LRNDescriptor(n, a, b, k) y = get_gpu(x).empty_like_me() with cu.cudnn_handler() as handle: cu.cuLocalResponseNormalizationForward(handle, lrn_desc, x, y) ret = cls._create_node(y) ret.attrs._x = x ret.attrs._lrn_desc = lrn_desc return ret
def _oper_gpu(cls, x, in_shape, out_shape, karnel, stride, padding): N = x.shape[0] pool_desc = cu.createPoolingDescriptor(karnel, padding, stride, pool_mode=1) y = GPUValue(shape=tuple([N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuPoolingForward(handle, pool_desc, x, y) ret = cls._create_node(y) ret.attrs._pool_desc = pool_desc ret.attrs._x = x return ret
def _backward_gpu(self, context, dy, **kwargs): if isinstance(self.attrs._arg, Node): with cu.cudnn_handler() as handle: dx = get_gpu(self).empty_like_me() cu.cuSoftmaxBackward(handle, get_gpu(self), get_gpu(dy), dx, mode=1) self.attrs._arg._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, lhs, rhs): N = lhs.shape[0] z = get_gpu(lhs).empty_like_me() tmp1 = get_gpu(lhs).empty_like_me() with cu.cudnn_handler() as handle: cu.cuSoftmaxForward(handle, lhs, z, mode=1) cu.cucross_entropy(get_gpu(z), get_gpu(rhs), get_gpu(tmp1)) loss = -cu.cusum(get_gpu(tmp1)) / N ret = cls._create_node(loss) ret.attrs._z = z ret.attrs._lhs = lhs ret.attrs._rhs = rhs return ret
def _backward_gpu(self, context, dy): dw, db, dx = (get_gpu(g).empty_like_me() for g in (self.attrs._w, self.attrs._b, self.attrs._x)) with cu.cudnn_handler() as handle: cu.cuConvolutionBackward(handle, self.attrs._conv_desc, self.attrs._filter_desc, self.attrs._x, self.attrs._w, dy, dw, db, dx) if isinstance(self.attrs._w, Node): self.attrs._w._update_diff(context, dw) if isinstance(self.attrs._x, Node): self.attrs._x._update_diff(context, dx) if isinstance(self.attrs._b, Node): self.attrs._b._update_diff(context, db)
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding): conv_desc = cu.ConvolutionDescriptor(padding, stride, precision) filter_desc = cu.FilterDescriptor(w.shape, precision) N = x.shape[0] # TODO: dirty code z = GPUValue(shape=tuple([N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuConvolutionBackwardData(handle, conv_desc, filter_desc, w, x, z) if b is not None: cu.cu_add_bias(get_gpu(b), z) ret = cls._create_node(z) ret.attrs._conv_desc = conv_desc ret.attrs._filter_desc = filter_desc ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b return ret
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding): N = x.shape[0] conv_desc = cu.createConvplutionDescriptor(padding, stride, precision) filter_desc = cu.createFilterDescriptor(w.shape, precision) # TODO: dirty code y = GPUValue(shape=tuple([N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuConvolutionForward(handle, conv_desc, filter_desc, x, w, y) if b is not None: cu.cuadd(get_gpu(y), get_gpu(b), get_gpu(y)) ret = cls._create_node(y) ret.attrs._conv_desc = conv_desc ret.attrs._filter_desc = filter_desc ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b return ret
def _oper_gpu(cls, x, w, b, momentum, mov_m, mov_s, inference, mode, epsilon): if mode == BATCH_NORMALIZE_FEATUREMAP: axs = 1 else: axs = 0 y, mean, sq_var = (get_gpu(g).empty_like_me() for g in (x, w, w)) mov_m = get_gpu(mov_m) mov_s = get_gpu(mov_s) mv_m = mov_m if isinstance(mov_m, GPUValue) else get_gpu(w).zeros_like_me() mv_v = mov_s if isinstance(mov_s, GPUValue) else get_gpu(w).zeros_like_me() with cu.cudnn_handler() as handle: cu.cuBatchNormalizatoinForward(handle, x, mv_m, mv_v, w, b, y, mean, sq_var, momentum=momentum, mode=axs, inference=inference, eps=epsilon) ret = cls._create_node(y) ret.attrs._axs = axs ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b ret.attrs._m = mean ret.attrs._v = sq_var if not inference: ret.attrs._mov_m = mv_m ret.attrs._mov_v = mv_v return ret
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding): N = x.shape[0] conv_desc = cu.ConvolutionDescriptor(padding, stride, precision) filter_desc = cu.FilterDescriptor(w.shape, precision) y = GPUValue(shape=tuple([ N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuConvolutionForward(handle, conv_desc, filter_desc, x, w, y) if b is not None: cu.cu_add_bias(get_gpu(b), y) # assert type(x) is not np.ndarray ret = cls._create_node(y) ret.attrs._conv_desc = conv_desc ret.attrs._filter_desc = filter_desc ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b return ret
def _oper_gpu(cls, arg): z = get_gpu(arg).empty_like_me() with cu.cudnn_handler() as handle: cu.cuSoftmaxForward(handle, arg, z, mode=1) return z