Beispiel #1
0
    def __backward(self, dy):
        dy = cp.array(dy)

        n_batches, _, _, _ = self.fire.shape
        n_channels, n_rows, n_cols = self.input_shape
        n_filters, n_rows_filter, n_cols_filter = self.filter_shape
        dy = dy.transpose(0, 2, 3, 1).reshape(-1, n_filters)

        input_shape = (n_batches, n_channels, n_rows, n_cols)
        backfire = np.dot(dy, cp.array(self.w[1:, :]).T)

        backfire = col2im(backfire,
                          input_shape,
                          self.output_shape,
                          self.filter_shape,
                          self.pad,
                          self.strides,
                          aggregate=True)

        if self.pad[0] > 0:
            backfire = backfire[:, :, self.pad[0]:-self.pad[0], :]
        if self.pad[1] > 0:
            backfire = backfire[:, :, :, self.pad[1]:-self.pad[1]]

        self.backfire = asnumpy(backfire)
        self.dw = asnumpy(
            self.dtype(1.) / n_batches * cp.dot(cp.array(self.x).T, dy))
Beispiel #2
0
    def __forward(self, x):
        mask = self.mask if self.force_cpu else cp.array(self.mask)
        x = x if self.force_cpu else cp.array(x)

        ncp.random_shuffle(mask.reshape(mask.size))
        mask_dropped = mask >= self.thresh
        fire = mask_dropped * x

        self.fire = asnumpy(fire)
        self.mask = asnumpy(mask)
        self.mask_dropped = asnumpy(mask_dropped)
Beispiel #3
0
    def optimize(self, w, dw):
        w_ = cp.array(w)
        dw_ = cp.array(dw)

        w_[1:, :] *= self.regularization(self.learning_rate, self.weight_decay)
        h = cp.zeros_like(w_) if self.h is None else cp.array(self.h)
        h = self.gamma * h + (1. - self.gamma) * cp.power(dw_, 2)
        w_ -= self.learning_rate * dw_ / (cp.sqrt(h) + self.ep)

        w[::] = asnumpy(w_)
        self.h = asnumpy(h)
Beispiel #4
0
    def __backward(self, dy):
        dy = cp.array(dy)
        if is_multi_channels_image(self.output_shape):
            dy = flatten(dy, self.output_shape)

        batch_size = self.x.shape[0]
        self.dw = asnumpy(
            self.dtype(1.) / batch_size * cp.dot(cp.array(self.x).T, dy))
        backfire = cp.dot(dy, cp.array(self.w[1:, :]).T)

        if is_multi_channels_image(self.input_shape):
            backfire = unflatten(backfire, self.input_shape)

        self.backfire = asnumpy(backfire)
Beispiel #5
0
    def __forward(self, x):
        x = cp.array(x)
        if is_multi_channels_image(self.input_shape):
            x = flatten(x, self.input_shape)

        # Add bias terms.
        x = cp.c_[cp.ones((x.shape[0], 1), dtype=self.dtype), x]
        fire = cp.dot(x, cp.array(self.w))

        if is_multi_channels_image(self.output_shape):
            fire = unflatten(fire, self.output_shape)

        self.x = asnumpy(x)
        self.fire = asnumpy(fire)
Beispiel #6
0
    def optimize(self, w, dw):
        w_ = cp.array(w)
        dw_ = cp.array(dw)

        w_[1:, :] *= self.regularization(self.learning_rate, self.weight_decay)

        pre_dw = cp.zeros_like(dw_) if self.pre_dw is None else cp.array(
            self.pre_dw)
        pre_dw = self.learning_rate * dw_ + self.momentum_rate * pre_dw

        w_ -= pre_dw

        w[::] = asnumpy(w_)
        self.pre_dw = asnumpy(pre_dw)
Beispiel #7
0
    def optimize(self, w, dw):
        w_ = cp.array(w)
        dw_ = cp.array(dw)

        w_[1:, :] *= self.regularization(self.learning_rate, self.weight_decay)
        w_ -= self.learning_rate * dw_

        w[::] = asnumpy(w_)
Beispiel #8
0
    def optimize(self, w, dw):
        w_ = cp.array(w)
        dw_ = cp.array(dw)

        w_[1:, :] *= self.regularization(self.learning_rate, self.weight_decay)
        v = cp.zeros_like(w_) if self.v is None else cp.array(self.v)
        r = cp.zeros_like(w_) if self.r is None else cp.array(self.r)

        dw_square = cp.power(dw_, 2)
        v = self.beta * (v - dw_) + dw_
        r = self.gamma * (r - dw_square) + dw_square

        w_ -= self.learning_rate / cp.sqrt(r + self.ep) * v

        w[::] = asnumpy(w_)
        self.v = asnumpy(v)
        self.r = asnumpy(r)
Beispiel #9
0
    def __predict(self, x):
        x = x if self.force_cpu else cp.array(x)
        gamma = self.gamma if self.force_cpu else cp.array(self.gamma)
        beta = self.beta if self.force_cpu else cp.array(self.beta)
        miu = self.miu if self.force_cpu else cp.array(self.miu)
        var = self.var if self.force_cpu else cp.array(self.var)

        fire = gamma * (x - miu) / ncp.sqrt(var + self.ep) + beta

        self.fire = asnumpy(fire)
Beispiel #10
0
    def optimize(self, w, dw):
        w_ = cp.array(w)
        dw_ = cp.array(dw)

        w_[1:, :] *= self.regularization(self.learning_rate, self.weight_decay)

        r = cp.zeros_like(w_) if self.r is None else cp.array(self.r)
        s = cp.zeros_like(w_) if self.s is None else cp.array(self.s)
        v = cp.zeros_like(w_) if self.v is None else cp.array(self.v)

        r = self.gamma * r + (1. - self.gamma) * cp.power(dw_, 2)
        v = cp.sqrt(s + self.ep) / (cp.sqrt(r + self.ep)) * dw_

        w_ -= self.learning_rate * v
        s = self.gamma + (1. - self.gamma) * cp.power(v, 2)

        w[::] = asnumpy(w_)
        self.r = asnumpy(r)
        self.s = asnumpy(s)
        self.v = asnumpy(v)
Beispiel #11
0
    def __forward(self, x):
        x = cp.array(x)

        if len(x.shape) != 4:
            msg = 'Convolution layer assumes that input is 4-d array.\n'\
                + '    shape : %s' % str(x.shape)
            raise DNNetRuntimeError(msg)

        n_batches, _, _, _ = x.shape
        n_channels, n_rows, n_cols = self.output_shape

        x_pad = pad_img(x, self.pad[0], self.pad[1])
        x = im2col(x_pad, self.filter_shape, self.strides)
        x = cp.c_[cp.ones((x.shape[0], 1), dtype=self.dtype), x]

        fire = cp.dot(x, cp.array(self.w))
        fire = fire.reshape(n_batches, n_rows, n_cols, n_channels)
        fire = fire.transpose(0, 3, 1, 2)

        self.x = asnumpy(x)
        self.fire = asnumpy(fire)
Beispiel #12
0
    def optimize(self, w, dw):
        w_ = cp.array(w)
        dw_ = cp.array(dw)

        w_[1:, :] *= self.regularization(self.learning_rate, self.weight_decay)
        h = cp.zeros_like(w_) if self.h is None else cp.array(self.h)

        h += cp.power(dw_, 2)
        w_ -= self.learning_rate * (self.dtype(1.) /
                                    cp.sqrt(h + self.ep)) * dw_

        w[::] = asnumpy(w_)
        self.h = h
Beispiel #13
0
    def __forward(self, x):
        x = x if self.force_cpu else cp.array(x)
        miu = ncp.mean(x, axis=0)
        xmiu = x - miu

        var = ncp.mean(xmiu**2, axis=0)
        std_inv = 1. / (ncp.sqrt(var + self.ep))

        gamma, beta = None, None
        shape = self.input_shape
        if self.gamma is None:
            gamma = ncp.ones(shape, dtype=self.dtype, arr_type=type(x))
        else:
            gamma = self.gamma if self.force_cpu else cp.array(self.gamma)
        if self.beta is None:
            beta = ncp.zeros(shape, dtype=self.dtype, arr_type=type(x))
        else:
            beta = self.beta if self.force_cpu else cp.array(self.beta)

        xhat = xmiu * std_inv
        fire = gamma * xhat + beta

        pre_miu, pre_var = None, None
        if self.miu is None:
            pre_miu = miu
        else:
            pre_miu = self.miu if self.force_cpu else cp.array(self.miu)
        if self.var is None:
            pre_var = var
        else:
            pre_var = self.var if self.force_cpu else cp.array(self.var)
        miu = pre_miu * self.momentum + (1. - self.momentum) * miu
        var = pre_var * self.momentum + (1. - self.momentum) * var

        self.xmiu = asnumpy(xmiu)
        self.var = asnumpy(var)
        self.std_inv = asnumpy(std_inv)
        self.gamma = asnumpy(gamma)
        self.beta = asnumpy(beta)
        self.xhat = asnumpy(xhat)
        self.fire = asnumpy(fire)
        self.miu = asnumpy(miu)
        self.var = asnumpy(var)
Beispiel #14
0
    def __backward(self, dy):
        dy = dy if self.force_cpu else cp.array(dy)
        xhat = self.xhat if self.force_cpu else cp.array(self.xhat)
        xmiu = self.xmiu if self.force_cpu else cp.array(self.xmiu)
        std_inv = self.std_inv if self.force_cpu else cp.array(self.std_inv)
        beta = self.beta if self.force_cpu else cp.array(self.beta)
        gamma = self.gamma if self.force_cpu else cp.array(self.gamma)

        batch_size = dy.shape[0]
        dbeta = dy.sum(axis=0)
        dgamma = (xhat * dy).sum(axis=0)

        tmp1 = (gamma * xmiu * dy).sum(axis=0)
        tmp2 = -ncp.power(std_inv, 3) * tmp1 / batch_size
        tmp3 = xmiu * tmp2 + gamma * std_inv * dy
        tmp4 = tmp3.sum(axis=0)

        backfire = tmp3 - tmp4 / batch_size
        beta = beta - dbeta / batch_size
        gamma = gamma - dgamma / batch_size

        self.backfire = asnumpy(backfire)
        self.beta = asnumpy(beta)
        self.gamma = asnumpy(gamma)
Beispiel #15
0
 def __predict(self, x):
     x = x if self.force_cpu else cp.array(x)
     self.fire = asnumpy((1. - self.drop_ratio) * x)
Beispiel #16
0
 def __backward(self, dy):
     mask_dropped = self.mask_dropped if self.force_cpu else cp.array(
         self.mask_dropped)
     dy = dy if self.force_cpu else cp.array(dy)
     self.backfire = asnumpy(mask_dropped * dy)