def _oper_gpu(cls, arg, slope): z = get_gpu(arg).empty_like_me() cu.culeaky_leru_forward(slope, get_gpu(arg), z) ret = cls._create_node(z) ret.attrs._arg = arg ret.attrs._slope = slope return ret
def _oper_gpu(cls, x, w): z = GPUValue(shape=(len(x), len(w[0]))) cu.cuembedding_forward(get_gpu(x), get_gpu(w), z) ret = cls._create_node(z) ret.attrs._x = x ret.attrs._w = w return ret
def _oper_gpu(cls, x, pz, ps, parameter): p = parameter s = get_gpu(np.zeros((x.shape[0], p["w"].shape[1]), dtype=precision)) if ps is None else ps z = get_gpu(s).zeros_like_me() if pz is None else pz u = dot(x, p["w"]) + dot(z, p["wr"]) + p["b"] gate_f = sigmoid(dot(x, p["wf"]) + dot(z, p["wfr"]) + p["wfc"] * s + p["bf"]) gate_i = sigmoid(dot(x, p["wi"]) + dot(z, p["wir"]) + p["wic"] * s + p["bi"]) state = gate_i * tanh(u) + gate_f * s gate_o = sigmoid( dot(x, p["wo"]) + dot(z, p["wor"]) + p["bo"] + p["woc"] * state) z = tanh(state) * gate_o ret = cls._create_node(get_gpu(z)) ret.attrs._x = x ret.attrs._p = parameter ret.attrs._u = u ret.attrs._pgated_f = None ret.attrs._pstate = ps ret.attrs._state = state ret.attrs._gated_o = gate_o ret.attrs._gated_f = gate_f ret.attrs._gated_i = gate_i ret.attrs._dt_d = [p[k] for k in ["wr", "wi", "wf", "wo", "w"]] ret._state = state return ret
def _backward_gpu(self, context, dy): if isinstance(self.attrs._arg, Node): alpha = self.attrs._alpha lmda = self.attrs._lmda dx = get_gpu(self.attrs._arg).empty_like_me() cu.cueru_backward(alpha, get_gpu(self.attrs._arg), dx) self.attrs._arg._update_diff(context, dx * get_gpu(dy) * lmda)
def _oper_gpu(cls, lhs, rhs): axis = 1 newshape = lhs.shape[:axis] + ( lhs.shape[axis] + rhs.shape[axis], ) + lhs.shape[axis + 1:] ret = GPUValue(shape=newshape) cuconcat(get_gpu(lhs), get_gpu(rhs), ret, axis) return ret
def _backward_gpu(self, context, dy, **kwargs): dx = get_gpu(self.attrs._x).empty_like_me() with cu.cudnn_handler() as handle: cu.cuPoolingBackward(handle, self.attrs._pool_desc, self.attrs._x, self, dy, dx) if isinstance(self.attrs._x, Node): self.attrs._x._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding): conv_desc = cu.createConvplutionDescriptor(padding, stride, precision) filter_desc = cu.createFilterDescriptor(w.shape, precision) N = x.shape[0] # TODO: dirty code z = GPUValue(shape=tuple([ N, ] + list(out_shape))) with cu.cudnn_handler() as handle: cu.cuConvolutionBackwardData(handle, conv_desc, filter_desc, w, x, z) cu.cuadd(get_gpu(z), get_gpu(b), get_gpu(z)) ret = cls._create_node(z) ret.attrs._conv_desc = conv_desc ret.attrs._filter_desc = filter_desc ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b return ret
def _oper_gpu(cls, x, w, b, momentum, mov_m, mov_s, inference, mode, epsilon): if mode == BATCH_NORMALIZE_FEATUREMAP: axs = 1 else: axs = 0 y, mean, sq_var = (get_gpu(g).empty_like_me() for g in (x, w, w)) mov_m = get_gpu(mov_m) mov_s = get_gpu(mov_s) mv_m = mov_m if isinstance(mov_m, GPUValue) else get_gpu(w).zeros_like_me() mv_v = mov_s if isinstance(mov_s, GPUValue) else get_gpu(w).zeros_like_me() with cu.cudnn_handler() as handle: cu.cuBatchNormalizatoinForward(handle, x, mv_m, mv_v, w, b, y, mean, sq_var, momentum=momentum, mode=axs, inference=inference, eps=epsilon) ret = cls._create_node(y) ret.attrs._axs = axs ret.attrs._x = x ret.attrs._w = w ret.attrs._b = b ret.attrs._m = mean ret.attrs._v = sq_var if not inference: ret.attrs._mov_m = mv_m ret.attrs._mov_v = mv_v return ret