def forward(self, x, gamma, beta): assert x.ndim == 2 or x.ndim == 4 x_ndim = x.ndim if x_ndim == 4: N, C, H, W = x.shape # (N, C, H, W) -> (N*H*W, C) x = x.transpose(0, 2, 3, 1).reshape(-1, C) xp = cuda.get_array_module(x) if kdezero.Config.train: mean = x.mean(axis=0) var = x.var(axis=0) inv_std = 1 / xp.sqrt(var + self.eps) xc = (x - mean) * inv_std m = x.size // gamma.size s = m - 1. if m - 1. > 1. else 1. adjust = m / s # unbiased estimation self.avg_mean *= self.decay self.avg_mean += (1 - self.decay) * mean self.avg_var *= self.decay self.avg_var += (1 - self.decay) * adjust * var self.inv_std = inv_std else: inv_std = 1 / xp.sqrt(self.avg_var + self.eps) xc = (x - self.avg_mean) * inv_std y = gamma * xc + beta if x_ndim == 4: # (N*H*W, C) -> (N, C, H, W) y = y.reshape(N, H, W, C).transpose(0, 3, 1, 2) return y
def forward(self, x, W, b): xp = cuda.get_array_module(x) Weight = W SH, SW = self.stride PH, PW = self.pad C, OC, KH, KW = Weight.shape N, C, H, W = x.shape if self.outsize is None: out_h = get_deconv_outsize(H, KH, SH, PH) out_w = get_deconv_outsize(W, KW, SW, PW) else: out_h, out_w = pair(self.outsize) img_shape = (N, OC, out_h, out_w) gcol = xp.tensordot(Weight, x, (0, 1)) gcol = xp.rollaxis(gcol, 3) y = col2im_array(gcol, img_shape, (KH, KW), self.stride, self.pad, to_matrix=False) if b is not None: self.no_bias = True y += b.reshape((1, b.size, 1, 1)) return y
def im2col_array(img, kernel_size, stride, pad, to_matrix=True): N, C, H, W = img.shape KH, KW = pair(kernel_size) SH, SW = pair(stride) PH, PW = pair(pad) OH = get_conv_outsize(H, KH, SH, PH) OW = get_conv_outsize(W, KW, SW, PW) xp = cuda.get_array_module(img) if xp != np: col = _im2col_gpu(img, kernel_size, stride, pad) else: img = np.pad(img, ((0, 0), (0, 0), (PH, PH + SH - 1), (PW, PW + SW - 1)), mode='constant', constant_values=(0, )) col = np.ndarray((N, C, KH, KW, OH, OW), dtype=img.dtype) for j in range(KH): j_lim = j + SH * OH for i in range(KW): i_lim = i + SW * OW col[:, :, j, i, :, :] = img[:, :, j:j_lim:SH, i:i_lim:SW] if to_matrix: col = col.transpose((0, 4, 5, 1, 2, 3)).reshape((N * OH * OW, -1)) return col
def forward(self, x): if self.W.data is None: self.in_size = x.shape[1] xp = cuda.get_array_module(x) self._init_W(xp) y = F.linear(x, self.W, self.b) return y
def forward(self, x): if self.W.data is None: self.in_channels = x.shape[1] xp = cuda.get_array_module(x) self._init_W(xp) y = F.deconv2d(x, self.W, self.b, self.stride, self.pad) return y
def forward(self, x, gy): xp = cuda.get_array_module(x) col = im2col_array(x, self.kernel_size, self.stride, self.pad, to_matrix=False) gW = xp.tensordot(gy, col, ((0, 2, 3), (0, 4, 5))) return gW
def forward(self, gy): xp = cuda.get_array_module(gy) gx = xp.zeros(self.in_shape, dtype=gy.dtype) if xp is np: np.add.at(gx, self.slices, gy) else: xp.scatter_add(gx, self.slices, gy) return gx
def backward(self, gy): x, t = self.inputs N, CLS_NUM = x.shape gy *= 1/N y = softmax(x) xp = cuda.get_array_module(t.data) t_onehot = xp.eye(CLS_NUM, dtype=t.dtype)[t.data] y = (y - t_onehot) * gy return y
def dropout(x, dropout_ratio=0.5): x = as_variable(x) if kdezero.Config.train: xp = cuda.get_array_module(x) mask = xp.random.rand(*x.shape) > dropout_ratio scale = xp.array(1.0 - dropout_ratio).astype(x.dtype) y = x * mask / scale return y else: return x
def forward(self, x, W, b): xp = cuda.get_array_module(x) KH, KW = W.shape[2:] col = im2col_array(x, (KH, KW), self.stride, self.pad, to_matrix=False) y = xp.tensordot(col, W, ((1, 2, 3), (1, 2, 3))) if b is not None: y += b y = xp.rollaxis(y, 3, 1) return y
def _init_params(self, x): xp = cuda.get_array_module(x) D = x.shape[1] if self.avg_mean.data is None: self.avg_mean.data = xp.zeros(D, dtype=x.dtype) if self.avg_var.data is None: self.avg_var.data = xp.ones(D, dtype=x.dtype) if self.gamma.data is None: self.gamma.data = xp.ones(D, dtype=x.dtype) if self.beta.data is None: self.beta.data = xp.zeros(D, dtype=x.dtype)
def logsumexp(x, axis=1): """After exp, sum elements along axes, and apply log. Args: x (ndarray): axis (int, optional): (default: 1) Returns: ndarray: """ xp = cuda.get_array_module(x) m = x.max(axis=axis, keepdims=True) y = x - m xp.exp(y, out=y) s = y.sum(axis=axis, keepdims=True) xp.log(s, out=s) m += s return m
def forward(self, gy): xp = cuda.get_array_module(gy) N, CH, OH, OW = gy.shape N, C, H, W = self.input_shape KH, KW = pair(self.kernel_size) gcol = xp.zeros((N * C * OH * OW * KH * KW), dtype=self.dtype) indexes = (self.indexes.ravel() + xp.arange(0, self.indexes.size * KH * KW, KH * KW)) gcol[indexes] = gy.ravel() gcol = gcol.reshape(N, C, OH, OW, KH, KW) gcol = xp.swapaxes(gcol, 2, 4) gcol = xp.swapaxes(gcol, 3, 5) gx = col2im_array(gcol, (N, C, H, W), self.kernel_size, self.stride, self.pad, to_matrix=False) return gx
def col2im_array(col, img_shape, kernel_size, stride, pad, to_matrix=True): N, C, H, W = img_shape KH, KW = pair(kernel_size) SH, SW = pair(stride) PH, PW = pair(pad) OH = get_conv_outsize(H, KH, SH, PH) OW = get_conv_outsize(W, KW, SW, PW) if to_matrix: col = col.reshape(N, OH, OW, C, KH, KW).transpose(0, 3, 4, 5, 1, 2) xp = cuda.get_array_module(col) if xp != np: img = _col2im_gpu(col, SH, SW, PH, PW, H, W) return img else: img = np.zeros((N, C, H + 2 * PH + SH - 1, W + 2 * PW + SW - 1), dtype=col.dtype) for j in range(KH): j_lim = j + SH * OH for i in range(KW): i_lim = i + SW * OW img[:, :, j:j_lim:SH, i:i_lim:SW] += col[:, :, j, i, :, :] return img[:, :, PH:H + PH, PW:W + PW]
def forward(self, x): xp = cuda.get_array_module(x) y = xp.maximum(x, 0.0) return y
def forward(self, x): xp = cuda.get_array_module(x) y = x - x.max(axis=self.axis, keepdims=True) y = xp.exp(y) y /= y.sum(axis=self.axis, keepdims=True) return y
def forward(self, x): self.x_shape = x.shape xp = cuda.get_array_module(x) y = xp.broadcast_to(x, self.shape) return y
def forward(self, x): xp = cuda.get_array_module(x) y = xp.tanh(x * 0.5) * 0.5 + 0.5 return y