def backward(self, dh_next, dc_next): Wx, Wh, _ = self.params x, h_prev, c_prev, i, f, g, o, c_next = self.cache tanh_c_next = np.tanh(c_next) ds = dc_next + (dh_next * o) * (1 - tanh_c_next**2) dc_prev = ds * f di = ds * g df = ds * c_prev do = dh_next * tanh_c_next dg = ds * i di *= i * (1 - i) df *= f * (1 - f) do *= o * (1 - o) dg *= (1 - g**2) dA = np.hstack((df, dg, di, do)) dWh = np.dot(h_prev.T, dA) dWx = np.dot(x.T, dA) db = dA.sum(axis=0) self.grads[0][...] = dWx self.grads[1][...] = dWh self.grads[2][...] = db dx = np.dot(dA, Wx.T) dh_prev = np.dot(dA, Wh.T) return dx, dh_prev, dc_prev
def analogy(a, b, c, word_to_id, id_to_word, word_matrix, top=5, answer=None): for word in (a, b, c): if word not in word_to_id: print('%s is not found' % word) return print('\n[analogy] ' + a + ':' + b + ' = ' + c + ':?') a_vec, b_vec, c_vec = word_matrix[word_to_id[a]], word_matrix[ word_to_id[b]], word_matrix[word_to_id[c]] query_vec = b_vec - a_vec + c_vec query_vec = normalize(query_vec) similarity = np.dot(word_matrix, query_vec) # cos similality # word_matrix_norm = np.sqrt(np.sum(word_matrix**2, axis=1)) # query_vec_norm = np.sqrt(np.sum(query_vec**2)) # norm = word_matrix_norm * query_vec_norm # similarity /= norm if answer is not None: print("==>" + answer + ":" + str(np.dot(word_matrix[word_to_id[answer]], query_vec))) count = 0 for i in (-1 * similarity).argsort(): if np.isnan(similarity[i]): continue if id_to_word[i] in (a, b, c): continue print(' {0}: {1}'.format(id_to_word[i], similarity[i])) count += 1 if count >= top: return
def forward(self, x, h_prev): Wx, Wh, b = self.params t = np.dot(h_prev, Wh) + np.dot(x, Wx) + b h_next = np.tanh(t) self.cache = (x, h_prev, h_next) return h_next
def backward(self, dout): W = self.params[0] dx = np.dot(dout, W.T) dW = np.dot(self.x.T, dout) db = np.sum(dout, axis=0) self.grads[0][...] = dW self.grads[1][...] = db return dx
def backward(self, dh_next): Wx, Wh, _ = self.params x, h_prev, h_next = self.cache dt = dh_next * (1 - h_next**2) db = np.sum(dt, axis=0) dWh = np.dot(h_prev.T, dt) dh_prev = np.dot(dt, Wh.T) dWx = np.dot(x.T, dt) dx = np.dot(dt, Wx.T) self.grads[0][...] = dWx self.grads[1][...] = dWh self.grads[2][...] = db return dx, dh_prev
def backward(self, dout): x = self.x N, T, D = x.shape W = self.params[0] dout = dout.reshape(N * T, -1) rx = x.reshape(N * T, D) db = np.sum(dout, axis=0) dW = np.dot(rx.T, dout) dx = np.dot(dout, W.T) dx = dx.reshape(N, T, D) self.grads[0][...] = dW self.grads[1][...] = db return dx
def forward(self, x): N, T, D = x.shape W, b = self.params rx = x.reshape(N * T, D) out = np.dot(rx, W) + b self.x = x return out.reshape(N, T, -1)
def cos_similarity(x, y, eps=1e-8): '''コサイン類似度の算出 :param x: ベクトル :param y: ベクトル :param eps: ”0割り”防止のための微小値 :return: ''' nx = x / (np.sqrt(np.sum(x**2)) + eps) ny = y / (np.sqrt(np.sum(y**2)) + eps) return np.dot(nx, ny)
def forward(self, x, h_prev, c_prev): Wx, Wh, b = self.params H = h_prev.shape[1] A = np.dot(x, Wx) + np.dot(h_prev, Wh) + b # slice f = A[:, :H] g = A[:, H:2 * H] i = A[:, 2 * H:3 * H] o = A[:, 3 * H:] f = sigmoid(f) g = np.tanh(g) i = sigmoid(i) o = sigmoid(o) c_next = f * c_prev + g * i h_next = o * np.tanh(c_next) self.cache = (x, h_prev, c_prev, i, f, g, o, c_next) return h_next, c_next
def forward(self, x): W, b = self.params out = np.dot(x, W) + b self.x = x return out
def backward(self, dout): W, = self.params dx = np.dot(dout, W.T) dW = np.dot(self.x.T, dout) self.grads[0][...] = dW return dx