def _backward_gpu(self, context, dy, **kwargs): gw, gx, gdy, gm, gv = map( get_gpu, (self.attrs._w, self.attrs._x, dy, self.attrs._m, self.attrs._v)) dx, dw, db = (g.ones_like_me() for g in (gx, gw, gw)) ax = self.attrs._axs with cu.cudnn_handler() as handle: cu.cuBatchNormalizatoinBackward(handle, gx, gw, gdy, gm, gv, dx, dw, db, mode=ax) if isinstance(self.attrs._x, Node): self.attrs._x._update_diff(context, dx, **kwargs) if isinstance(self.attrs._w, Node): self.attrs._w._update_diff(context, dw, **kwargs) if isinstance(self.attrs._b, Node): self.attrs._b._update_diff(context, db, **kwargs)
def _oper_gpu(cls, x, w, b, in_shape, kernel, stride, padding): conv_desc = cu.ConvolutionNDescriptor(padding, stride, precision) filter_desc = cu.NdFilterDescriptor(w.shape, precision) output_shape = [x.shape[0], w.shape[0]] for i in range(len(x.shape[2:])): output_shape.append( (x.shape[i + 2] + padding[i] * 2 - kernel[i]) // stride[i] + 1) y = GPUValue(shape=tuple(output_shape)) with cu.cudnn_handler() as handle: cu.cuConvolutionForward(handle, conv_desc, filter_desc, get_gpu(x), get_gpu(w), y) if b is not None: cu.cu_add_bias(get_gpu(b), y) # assert type(x) is not np.ndarray ret = cls._create_node(y) ret.attrs._conv_desc = conv_desc ret.attrs._filter_desc = filter_desc ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b return ret
def _backward_gpu(self, context, dy, **kwargs): if isinstance(self.attrs._x, Node): dx = get_gpu(self).empty_like_me() with cu.cudnn_handler() as handle: cu.cuLocalResponseNormalizationBackward( handle, self.attrs._lrn_desc, get_gpu(self.attrs._x), get_gpu(self), dx, get_gpu(dy)) self.attrs._x._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding, dilation): N = x.shape[0] conv_desc = cu.ConvolutionDescriptor(padding, stride, dilation, precision) filter_desc = cu.FilterDescriptor(w.shape, precision) y = GPUValue(shape=tuple([ N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuConvolutionForward(handle, conv_desc, filter_desc, get_gpu(x), get_gpu(w), y) if b is not None: cu.cu_add_bias(get_gpu(b), y) # assert type(x) is not np.ndarray ret = cls._create_node(y) ret.attrs._conv_desc = conv_desc ret.attrs._filter_desc = filter_desc ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b ret.attrs._in_shape = in_shape ret.attrs._out_shape = out_shape ret.attrs._kernel = kernel ret.attrs._stride = stride ret.attrs._padding = padding ret.attrs._dilation = dilation return ret
def _backward_gpu(self, context, dy, **kwargs): dx = get_gpu(self.attrs._x).empty_like_me() with cu.cudnn_handler() as handle: cu.cuPoolingBackward(handle, self.attrs._pool_desc, get_gpu(self.attrs._x), get_gpu(self), get_gpu(dy), dx) if isinstance(self.attrs._x, Node): self.attrs._x._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, x, n, k, a, b): lrn_desc = cu.LRNDescriptor(n, a, b, k) y = get_gpu(x).empty_like_me() with cu.cudnn_handler() as handle: cu.cuLocalResponseNormalizationForward(handle, lrn_desc, get_gpu(x), get_gpu(y)) ret = cls._create_node(y) ret.attrs._x = x ret.attrs._lrn_desc = lrn_desc return ret
def _backward_gpu(self, context, dy, **kwargs): if isinstance(self.attrs._arg, Node): with cu.cudnn_handler() as handle: dx = get_gpu(self).empty_like_me() cu.cuSoftmaxBackward(handle, get_gpu(self), get_gpu(dy), dx, mode=1) self.attrs._arg._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, x, prev_pool): dx = GPUValue(shape=prev_pool.attrs._x.shape) with cu.cudnn_handler() as handle: cu.cuPoolingBackward(handle, prev_pool.attrs._pool_desc, get_gpu( prev_pool.attrs._x), get_gpu(prev_pool), get_gpu(x), dx) ret = cls._create_node(dx) ret.attrs._x = x ret.attrs._original_x = prev_pool.attrs._x ret.attrs._kernel = prev_pool.attrs._kernel ret.attrs._stride = prev_pool.attrs._stride ret.attrs._padding = prev_pool.attrs._padding return ret
def _oper_gpu(cls, x, w, b, momentum, mov_m, mov_s, inference, mode, epsilon): if mode == BATCH_NORMALIZE_FEATUREMAP: axs = 1 else: axs = 0 if b is None: b = get_gpu(w).zeros_like_me() y, mean, sq_var = (get_gpu(g).empty_like_me() for g in (x, w, w)) if inference: inv_var = 1.0 / np.sqrt(to_value(mov_s) + epsilon) if isinstance(inv_var, Number): inv_var = inv_var * np.ones_like(w) mov_m = get_gpu(mov_m) mov_s = get_gpu(mov_s) mv_m = mov_m if isinstance(mov_m, GPUValue) else get_gpu(w).zeros_like_me() mv_v = mov_s if isinstance(mov_s, GPUValue) else get_gpu(w).zeros_like_me() with cu.cudnn_handler() as handle: cu.cuBatchNormalizatoinForward(handle, get_gpu(x), mv_m, mv_v, get_gpu(w), get_gpu(b), y, mean, sq_var, momentum=momentum, mode=axs, inference=inference, eps=epsilon) ret = cls._create_node(y) ret.attrs._axs = axs ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b if inference: ret.attrs._m = mv_m ret.attrs._v = inv_var else: ret.attrs._m = mean ret.attrs._v = sq_var ret.attrs._mov_m = mv_m ret.attrs._mov_v = mv_v return ret
def _oper_gpu(cls, x, in_shape, out_shape, karnel, stride, padding): N = x.shape[0] pool_desc = cu.PoolingDescriptor(karnel, padding, stride, pool_mode=1) y = GPUValue(shape=tuple([ N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuPoolingForward(handle, pool_desc, get_gpu(x), y) ret = cls._create_node(y) ret.attrs._pool_desc = pool_desc ret.attrs._kernel = karnel ret.attrs._stride = stride ret.attrs._padding = padding ret.attrs._x = x return ret
def _backward_gpu(self, context, dy, **kwargs): dw, db, dx = (get_gpu(g).empty_like_me() if g is not None else None for g in (self.attrs._w, self.attrs._b, self.attrs._x)) with cu.cudnn_handler() as handle: cu.cuConvolutionBackward(handle, self.attrs._conv_desc, self.attrs._filter_desc, get_gpu(self.attrs._x), get_gpu( self.attrs._w), get_gpu(dy), dw, db, dx, **kwargs) if isinstance(self.attrs._w, Node): self.attrs._w._update_diff(context, dw, **kwargs) if isinstance(self.attrs._x, Node): self.attrs._x._update_diff(context, dx, **kwargs) if isinstance(self.attrs._b, Node): self.attrs._b._update_diff(context, db, **kwargs)
def _oper_gpu(cls, x, karnel, stride, padding): pool_desc = cu.PoolingNDescriptor(karnel, padding, stride, pool_mode=1) output_shape = [x.shape[0], x.shape[1]] for i in range(len(x.shape[2:])): output_shape.append( (x.shape[i + 2] + padding[i] * 2 - karnel[i]) // stride[i] + 1) y = GPUValue(shape=tuple(output_shape)) with cu.cudnn_handler() as handle: cu.cuPoolingForward(handle, pool_desc, get_gpu(x), get_gpu(y)) ret = cls._create_node(y) ret.attrs._pool_desc = pool_desc ret.attrs._kernel = karnel ret.attrs._stride = stride ret.attrs._padding = padding ret.attrs._x = x return ret
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding, dilation): conv_desc = cu.ConvolutionDescriptor(padding, stride, dilation, precision) filter_desc = cu.FilterDescriptor(w.shape, precision) N = x.shape[0] z = GPUValue(shape=tuple([ N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuConvolutionBackwardData(handle, conv_desc, filter_desc, get_gpu(w), get_gpu(x), z) if b is not None: cu.cu_add_bias(get_gpu(b), z) ret = cls._create_node(z) ret.attrs._conv_desc = conv_desc ret.attrs._filter_desc = filter_desc ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b return ret
def _oper_gpu(cls, arg): z = get_gpu(arg).empty_like_me() with cu.cudnn_handler() as handle: cu.cuSoftmaxForward(handle, get_gpu(arg), z, mode=1) return z