def _oper_gpu(cls, arg, slope):
z = get_gpu(arg).empty_like_me()
cu.culeaky_leru_forward(slope, get_gpu(arg), z)
ret = cls._create_node(z)
ret.attrs._arg = arg
ret.attrs._slope = slope
return ret
def _oper_gpu(cls, x, w):
z = GPUValue(shape=(len(x), len(w[0])))
cu.cuembedding_forward(get_gpu(x), get_gpu(w), z)
ret = cls._create_node(z)
ret.attrs._x = x
ret.attrs._w = w
return ret
def _oper_gpu(cls, x, pz, ps, parameter):
p = parameter
s = get_gpu(np.zeros((x.shape[0], p["w"].shape[1]), dtype=precision)) if ps is None else ps
z = get_gpu(s).zeros_like_me() if pz is None else pz
u = dot(x, p["w"]) + dot(z, p["wr"]) + p["b"]
gate_f = sigmoid(dot(x, p["wf"]) +
dot(z, p["wfr"]) + p["wfc"] * s + p["bf"])
gate_i = sigmoid(dot(x, p["wi"]) +
dot(z, p["wir"]) + p["wic"] * s + p["bi"])
state = gate_i * tanh(u) + gate_f * s
gate_o = sigmoid(
dot(x, p["wo"]) + dot(z, p["wor"]) + p["bo"] + p["woc"] * state)
z = tanh(state) * gate_o
ret = cls._create_node(get_gpu(z))
ret.attrs._x = x
ret.attrs._p = parameter
ret.attrs._u = u
ret.attrs._pgated_f = None
ret.attrs._pstate = ps
ret.attrs._state = state
ret.attrs._gated_o = gate_o
ret.attrs._gated_f = gate_f
ret.attrs._gated_i = gate_i
ret.attrs._dt_d = [p[k] for k in ["wr", "wi", "wf", "wo", "w"]]
ret._state = state
return ret
def _backward_gpu(self, context, dy):
if isinstance(self.attrs._arg, Node):
alpha = self.attrs._alpha
lmda = self.attrs._lmda
dx = get_gpu(self.attrs._arg).empty_like_me()
cu.cueru_backward(alpha, get_gpu(self.attrs._arg), dx)
self.attrs._arg._update_diff(context, dx * get_gpu(dy) * lmda)
def _oper_gpu(cls, lhs, rhs):
axis = 1
newshape = lhs.shape[:axis] + (
lhs.shape[axis] + rhs.shape[axis], ) + lhs.shape[axis + 1:]
ret = GPUValue(shape=newshape)
cuconcat(get_gpu(lhs), get_gpu(rhs), ret, axis)
return ret
def _backward_gpu(self, context, dy, **kwargs):
dx = get_gpu(self.attrs._x).empty_like_me()
with cu.cudnn_handler() as handle:
cu.cuPoolingBackward(handle, self.attrs._pool_desc, self.attrs._x,
self, dy, dx)
if isinstance(self.attrs._x, Node):
self.attrs._x._update_diff(context, dx, **kwargs)
def _oper_gpu(cls, x, w, b, in_shape, out_shape, kernel, stride, padding):
conv_desc = cu.createConvplutionDescriptor(padding, stride, precision)
filter_desc = cu.createFilterDescriptor(w.shape, precision)
N = x.shape[0]
# TODO: dirty code
z = GPUValue(shape=tuple([
N,
] + list(out_shape)))
with cu.cudnn_handler() as handle:
cu.cuConvolutionBackwardData(handle, conv_desc, filter_desc, w, x,
z)
cu.cuadd(get_gpu(z), get_gpu(b), get_gpu(z))
ret = cls._create_node(z)
ret.attrs._conv_desc = conv_desc
ret.attrs._filter_desc = filter_desc
ret.attrs._x = x
ret.attrs._w = w
ret.attrs._b = b
return ret
def _oper_gpu(cls, x, w, b, momentum, mov_m, mov_s, inference, mode,
epsilon):
if mode == BATCH_NORMALIZE_FEATUREMAP:
axs = 1
else:
axs = 0
y, mean, sq_var = (get_gpu(g).empty_like_me() for g in (x, w, w))
mov_m = get_gpu(mov_m)
mov_s = get_gpu(mov_s)
mv_m = mov_m if isinstance(mov_m,
GPUValue) else get_gpu(w).zeros_like_me()
mv_v = mov_s if isinstance(mov_s,
GPUValue) else get_gpu(w).zeros_like_me()
with cu.cudnn_handler() as handle:
cu.cuBatchNormalizatoinForward(handle,
x,
mv_m,
mv_v,
w,
b,
y,
mean,
sq_var,
momentum=momentum,
mode=axs,
inference=inference,
eps=epsilon)
ret = cls._create_node(y)
ret.attrs._axs = axs
ret.attrs._x = x
ret.attrs._w = w
ret.attrs._b = b
ret.attrs._m = mean
ret.attrs._v = sq_var
if not inference:
ret.attrs._mov_m = mv_m
ret.attrs._mov_v = mv_v
return ret
