def __init__(self, Wx, Wh, b, stateful=False): self.params = [Wx, Wh, b] self.grads = [np.zeros_like(Wx), np.zeros_like(Wh), np.zeros_like(b)] self.layers = None self.h, self.dh = None, None self.stateful = stateful
def __init__(self, Wx, Wh, b): ''' Parameters ---------- Wx: 入力`x`用の重みパラメータ(4つ分の重みをまとめる) Wh: 隠れ状態`h`用の重みパラメータ(4つ分の重みをまとめる) b: バイアス(4つ分のバイアスをまとめる) ''' self.params = [Wx, Wh, b] self.grads = [np.zeros_like(Wx), np.zeros_like(Wh), np.zeros_like(b)] self.cache = None
def update(self, params, grads): if not self.m: self.m, self.v = [], [] for param in params: self.m.append(np.zeros_like(param)) self.v.append(np.zeros_like(param)) self.iter += 1 lr_t = self.lr * np.sqrt(1.0 - self.beta2 ** self.iter) / (1.0 - self.beta1 ** self.iter) for i in range(len(params)): self.m[i] += (1 - self.beta1) * (grads[i] - self.m[i]) self.v[i] += (1 - self.beta2) * (grads[i] ** 2 - self.v[i]) params[i] -= lr_t * self.m[i] / (np.sqrt(self.v[i]) + 1e-7)
def backward(self, dh): dhs = np.zeros_like(self.hs) dhs[:, -1, :] = dh dout = self.lstm.backward(dhs) dout = self.embed.backward(dout) return dout
def __init__(self, W): self.params = [W] self.grads = [np.zeros_like(W)] self.layers = None self.W = W
def __init__(self, Wx, Wh, b): self.params = [Wx, Wh, b] self.grads = [np.zeros_like(Wx), np.zeros_like(Wh), np.zeros_like(b)] self.cache = None
def __init__(self, W, b): self.params = [W, b] self.grads = [np.zeros_like(W), np.zeros_like(b)] self.x = None
def __init__(self, W): self.params = [W] self.grads = [np.zeros_like(W)] self.x = None