def forward(self, x, h_prev): Wx, Wh, b = self.params t = np.dot(h_prev, Wh) + np.dot(x, Wx) + b h_next = np.tanh(t) self.cache = (x, h_prev, h_next) return h_next
def backward(self, dh_next, dc_next): Wx, Wh, b = self.params x, h_prev, c_prev, i, f, g, o, c_next = self.cache tanh_c_next = np.tanh(c_next) ds = dc_next + (dh_next * o) * (1 - tanh_c_next ** 2) dc_prev = ds * f di = ds * g df = ds * c_prev do = dh_next * tanh_c_next dg = ds * i di *= i * (1 - i) df *= f * (1 - f) do *= o * (1 - o) dg *= (1 - g ** 2) dA = np.hstack((df, dg, di, do)) dWh = np.dot(h_prev.T, dA) dWx = np.dot(x.T, dA) db = dA.sum(axis=0) self.grads[0][...] = dWx self.grads[1][...] = dWh self.grads[2][...] = db dx = np.dot(dA, Wx.T) dh_prev = np.dot(dA, Wh.T) return dx, dh_prev, dc_prev
def backward(self, dout): W, _ = self.params dx = np.dot(dout, W.T) dW = np.dot(self.x.T, dout) db = np.sum(dout, axis=0) self.grads[0][...] = dW self.grads[1][...] = db return dx
def backward(self, dh_next): Wx, Wh, _ = self.params x, h_prev, h_next = self.cache dt = dh_next * (1 - h_next**2) db = np.sum(dt, axis=0) dWh = np.dot(h_prev.T, dt) dh_prev = np.dot(dt, Wh.T) dWx = np.dot(x.T, dt) dx = np.dot(dt, Wx.T) self.grads[0][...] = dWx self.grads[1][...] = dWh self.grads[2][...] = db return dx, dh_prev
def backward(self, dout): x = self.x N, T, D = x.shape W, b = self.params dout = dout.reshape(N * T, -1) rx = x.reshape(N * T, -1) db = np.sum(dout, axis=0) dW = np.dot(rx.T, dout) dx = np.dot(dout, W.T) dx = dx.reshape(*x.shape) self.grads[0][...] = dW self.grads[1][...] = db return dx
def forward(self, x): N, T, D = x.shape W, b = self.params rx = x.reshape(N * T, -1) out = np.dot(rx, W) + b self.x = x return out.reshape(N, T, -1)
def forward(self, x, h_prev, c_prev): Wx, Wh, b = self.params N, H = h_prev.shape A = np.dot(x, Wx) + np.dot(h_prev, Wh) + b f = A[:, :H] g = A[:, H:2*H] i = A[:, 2*H:3*H] o = A[:, 3*H:] f = sigmoid(f) g = np.tanh(g) i = sigmoid(i) o = sigmoid(o) c_next = f * c_prev + g * i h_next = o * np.tanh(c_next) self.cache = (x, h_prev, c_prev, i, f, g, o, c_next) return h_next, c_next
def forward(self, x, h_prev): H, _ = self.Wh.shape Wxz, Wxr, Wx = self.Wx[:, :H], self.Wx[:, H:2 * H], self.Wx[:, 2 * H:] Whz, Whr, Wh = self.Wh[:, :H], self.Wh[:, H:2 * H], self.Wh[:, 2 * H:] z = sigmoid(np.dot(x, Wxz) + np.dot(h_prev, Whz)) r = sigmoid(np.dot(x, Wxr) + np.dot(h_prev, Whr)) h_hat = np.tanh(np.dot(x, Wx) + np.dot(r * h_prev, Wh)) h_next = (1 - z) * h_prev + z * h_hat self.cache = (x, h_prev, z, r, h_hat) return h_next
def backward(self, dh_next): H, _ = self.Wh.shape Wxz, Wxr, Wx = self.Wx[:, :H], self.Wx[:, H:2 * H], self.Wx[:, 2 * H:] Whz, Whr, Wh = self.Wh[:, :H], self.Wh[:, H:2 * H], self.Wh[:, 2 * H:] x, h_prev, z, r, h_hat = self.cache dh_hat = dh_next * z dh_prev = dh_next * (1 - z) # tanh dt = dh_hat * (1 - h_hat**2) dWh = np.dot((r * h_prev).T, dt) dhr = np.dot(dt, Wh.T) dWx = np.dot(x.T, dt) dx = np.dot(dt, Wx.T) dh_prev += r * dhr # update gate(z) dz = dh_next * h_hat - dh_next * h_prev dt = dz * z * (1 - z) dWhz = np.dot(h_prev.T, dt) dh_prev += np.dot(dt, Whz.T) dWxz = np.dot(x.T, dt) dx += np.dot(dt, Wxz.T) # rest gate(r) dr = dhr * h_prev dt = dr * r * (1 - r) dWhr = np.dot(h_prev.T, dt) dh_prev += np.dot(dt, Whr.T) dWxr = np.dot(x.T, dt) dx += np.dot(dt, Wxr.T) self.dWx = np.hstack((dWxz, dWxr, dWx)) self.dWh = np.hstack((dWhz, dWhr, dWh)) return dx, dh_prev
def cos_similarity(x, y): nx = x / (numpy.sqrt(np.sum(x**2)) + 1e-8) ny = y / (numpy.sqrt(np.sum(y**2)) + 1e-8) return np.dot(nx, ny)
def forward(self, x): W, b = self.params out = np.dot(x, W) + b self.x = x return out
def backward(self, dout): W, = self.params dx = np.dot(dout, W.T) dW = np.dot(self.x.T, dout) self.grads[0][...] = dW return dx