def forward(self, x, gamma, beta):
assert x.ndim == 2 or x.ndim == 4
x_ndim = x.ndim
if x_ndim == 4:
N, C, H, W = x.shape
# (N, C, H, W) -> (N*H*W, C)
x = x.transpose(0, 2, 3, 1).reshape(-1, C)
xp = cuda.get_array_module(x)
if kdezero.Config.train:
mean = x.mean(axis=0)
var = x.var(axis=0)
inv_std = 1 / xp.sqrt(var + self.eps)
xc = (x - mean) * inv_std
m = x.size // gamma.size
s = m - 1. if m - 1. > 1. else 1.
adjust = m / s # unbiased estimation
self.avg_mean *= self.decay
self.avg_mean += (1 - self.decay) * mean
self.avg_var *= self.decay
self.avg_var += (1 - self.decay) * adjust * var
self.inv_std = inv_std
else:
inv_std = 1 / xp.sqrt(self.avg_var + self.eps)
xc = (x - self.avg_mean) * inv_std
y = gamma * xc + beta
if x_ndim == 4:
# (N*H*W, C) -> (N, C, H, W)
y = y.reshape(N, H, W, C).transpose(0, 3, 1, 2)
return y
def forward(self, x, W, b):
xp = cuda.get_array_module(x)
Weight = W
SH, SW = self.stride
PH, PW = self.pad
C, OC, KH, KW = Weight.shape
N, C, H, W = x.shape
if self.outsize is None:
out_h = get_deconv_outsize(H, KH, SH, PH)
out_w = get_deconv_outsize(W, KW, SW, PW)
else:
out_h, out_w = pair(self.outsize)
img_shape = (N, OC, out_h, out_w)
gcol = xp.tensordot(Weight, x, (0, 1))
gcol = xp.rollaxis(gcol, 3)
y = col2im_array(gcol,
img_shape, (KH, KW),
self.stride,
self.pad,
to_matrix=False)
if b is not None:
self.no_bias = True
y += b.reshape((1, b.size, 1, 1))
return y
def im2col_array(img, kernel_size, stride, pad, to_matrix=True):
N, C, H, W = img.shape
KH, KW = pair(kernel_size)
SH, SW = pair(stride)
PH, PW = pair(pad)
OH = get_conv_outsize(H, KH, SH, PH)
OW = get_conv_outsize(W, KW, SW, PW)
xp = cuda.get_array_module(img)
if xp != np:
col = _im2col_gpu(img, kernel_size, stride, pad)
else:
img = np.pad(img,
((0, 0), (0, 0), (PH, PH + SH - 1), (PW, PW + SW - 1)),
mode='constant',
constant_values=(0, ))
col = np.ndarray((N, C, KH, KW, OH, OW), dtype=img.dtype)
for j in range(KH):
j_lim = j + SH * OH
for i in range(KW):
i_lim = i + SW * OW
col[:, :, j, i, :, :] = img[:, :, j:j_lim:SH, i:i_lim:SW]
if to_matrix:
col = col.transpose((0, 4, 5, 1, 2, 3)).reshape((N * OH * OW, -1))
return col
def forward(self, x):
if self.W.data is None:
self.in_size = x.shape[1]
xp = cuda.get_array_module(x)
self._init_W(xp)
y = F.linear(x, self.W, self.b)
return y
def forward(self, x):
if self.W.data is None:
self.in_channels = x.shape[1]
xp = cuda.get_array_module(x)
self._init_W(xp)
y = F.deconv2d(x, self.W, self.b, self.stride, self.pad)
return y
def forward(self, x, gy):
xp = cuda.get_array_module(x)
col = im2col_array(x,
self.kernel_size,
self.stride,
self.pad,
to_matrix=False)
gW = xp.tensordot(gy, col, ((0, 2, 3), (0, 4, 5)))
return gW
def forward(self, gy):
xp = cuda.get_array_module(gy)
gx = xp.zeros(self.in_shape, dtype=gy.dtype)
if xp is np:
np.add.at(gx, self.slices, gy)
else:
xp.scatter_add(gx, self.slices, gy)
return gx
def backward(self, gy):
x, t = self.inputs
N, CLS_NUM = x.shape
gy *= 1/N
y = softmax(x)
xp = cuda.get_array_module(t.data)
t_onehot = xp.eye(CLS_NUM, dtype=t.dtype)[t.data]
y = (y - t_onehot) * gy
return y
def dropout(x, dropout_ratio=0.5):
x = as_variable(x)
if kdezero.Config.train:
xp = cuda.get_array_module(x)
mask = xp.random.rand(*x.shape) > dropout_ratio
scale = xp.array(1.0 - dropout_ratio).astype(x.dtype)
y = x * mask / scale
return y
else:
return x
def forward(self, x, W, b):
xp = cuda.get_array_module(x)
KH, KW = W.shape[2:]
col = im2col_array(x, (KH, KW), self.stride, self.pad, to_matrix=False)
y = xp.tensordot(col, W, ((1, 2, 3), (1, 2, 3)))
if b is not None:
y += b
y = xp.rollaxis(y, 3, 1)
return y
def _init_params(self, x):
xp = cuda.get_array_module(x)
D = x.shape[1]
if self.avg_mean.data is None:
self.avg_mean.data = xp.zeros(D, dtype=x.dtype)
if self.avg_var.data is None:
self.avg_var.data = xp.ones(D, dtype=x.dtype)
if self.gamma.data is None:
self.gamma.data = xp.ones(D, dtype=x.dtype)
if self.beta.data is None:
self.beta.data = xp.zeros(D, dtype=x.dtype)
def logsumexp(x, axis=1):
"""After exp, sum elements along axes, and apply log.
Args:
x (ndarray):
axis (int, optional): (default: 1)
Returns:
ndarray:
"""
xp = cuda.get_array_module(x)
m = x.max(axis=axis, keepdims=True)
y = x - m
xp.exp(y, out=y)
s = y.sum(axis=axis, keepdims=True)
xp.log(s, out=s)
m += s
return m
def forward(self, gy):
xp = cuda.get_array_module(gy)
N, CH, OH, OW = gy.shape
N, C, H, W = self.input_shape
KH, KW = pair(self.kernel_size)
gcol = xp.zeros((N * C * OH * OW * KH * KW), dtype=self.dtype)
indexes = (self.indexes.ravel() +
xp.arange(0, self.indexes.size * KH * KW, KH * KW))
gcol[indexes] = gy.ravel()
gcol = gcol.reshape(N, C, OH, OW, KH, KW)
gcol = xp.swapaxes(gcol, 2, 4)
gcol = xp.swapaxes(gcol, 3, 5)
gx = col2im_array(gcol, (N, C, H, W),
self.kernel_size,
self.stride,
self.pad,
to_matrix=False)
return gx
def col2im_array(col, img_shape, kernel_size, stride, pad, to_matrix=True):
N, C, H, W = img_shape
KH, KW = pair(kernel_size)
SH, SW = pair(stride)
PH, PW = pair(pad)
OH = get_conv_outsize(H, KH, SH, PH)
OW = get_conv_outsize(W, KW, SW, PW)
if to_matrix:
col = col.reshape(N, OH, OW, C, KH, KW).transpose(0, 3, 4, 5, 1, 2)
xp = cuda.get_array_module(col)
if xp != np:
img = _col2im_gpu(col, SH, SW, PH, PW, H, W)
return img
else:
img = np.zeros((N, C, H + 2 * PH + SH - 1, W + 2 * PW + SW - 1),
dtype=col.dtype)
for j in range(KH):
j_lim = j + SH * OH
for i in range(KW):
i_lim = i + SW * OW
img[:, :, j:j_lim:SH, i:i_lim:SW] += col[:, :, j, i, :, :]
return img[:, :, PH:H + PH, PW:W + PW]
def forward(self, x):
xp = cuda.get_array_module(x)
y = xp.maximum(x, 0.0)
return y
def forward(self, x):
xp = cuda.get_array_module(x)
y = x - x.max(axis=self.axis, keepdims=True)
y = xp.exp(y)
y /= y.sum(axis=self.axis, keepdims=True)
return y
def forward(self, x):
self.x_shape = x.shape
xp = cuda.get_array_module(x)
y = xp.broadcast_to(x, self.shape)
return y
def forward(self, x):
xp = cuda.get_array_module(x)
y = xp.tanh(x * 0.5) * 0.5 + 0.5
return y
