def backward(self, dh_next, dc_next):
Wx, Wh, _ = self.params
x, h_prev, c_prev, i, f, g, o, c_next = self.cache
tanh_c_next = np.tanh(c_next)
ds = dc_next + (dh_next * o) * (1 - tanh_c_next**2)
dc_prev = ds * f
di = ds * g
df = ds * c_prev
do = dh_next * tanh_c_next
dg = ds * i
di *= i * (1 - i)
df *= f * (1 - f)
do *= o * (1 - o)
dg *= (1 - g**2)
dA = np.hstack((df, dg, di, do))
dWh = np.dot(h_prev.T, dA)
dWx = np.dot(x.T, dA)
db = dA.sum(axis=0)
self.grads[0][...] = dWx
self.grads[1][...] = dWh
self.grads[2][...] = db
dx = np.dot(dA, Wx.T)
dh_prev = np.dot(dA, Wh.T)
return dx, dh_prev, dc_prev
def analogy(a, b, c, word_to_id, id_to_word, word_matrix, top=5, answer=None):
for word in (a, b, c):
if word not in word_to_id:
print('%s is not found' % word)
return
print('\n[analogy] ' + a + ':' + b + ' = ' + c + ':?')
a_vec, b_vec, c_vec = word_matrix[word_to_id[a]], word_matrix[
word_to_id[b]], word_matrix[word_to_id[c]]
query_vec = b_vec - a_vec + c_vec
query_vec = normalize(query_vec)
similarity = np.dot(word_matrix, query_vec)
# cos similality
# word_matrix_norm = np.sqrt(np.sum(word_matrix**2, axis=1))
# query_vec_norm = np.sqrt(np.sum(query_vec**2))
# norm = word_matrix_norm * query_vec_norm
# similarity /= norm
if answer is not None:
print("==>" + answer + ":" +
str(np.dot(word_matrix[word_to_id[answer]], query_vec)))
count = 0
for i in (-1 * similarity).argsort():
if np.isnan(similarity[i]):
continue
if id_to_word[i] in (a, b, c):
continue
print(' {0}: {1}'.format(id_to_word[i], similarity[i]))
count += 1
if count >= top:
return
def forward(self, x, h_prev):
Wx, Wh, b = self.params
t = np.dot(h_prev, Wh) + np.dot(x, Wx) + b
h_next = np.tanh(t)
self.cache = (x, h_prev, h_next)
return h_next
def backward(self, dout):
W = self.params[0]
dx = np.dot(dout, W.T)
dW = np.dot(self.x.T, dout)
db = np.sum(dout, axis=0)
self.grads[0][...] = dW
self.grads[1][...] = db
return dx
def backward(self, dh_next):
Wx, Wh, _ = self.params
x, h_prev, h_next = self.cache
dt = dh_next * (1 - h_next**2)
db = np.sum(dt, axis=0)
dWh = np.dot(h_prev.T, dt)
dh_prev = np.dot(dt, Wh.T)
dWx = np.dot(x.T, dt)
dx = np.dot(dt, Wx.T)
self.grads[0][...] = dWx
self.grads[1][...] = dWh
self.grads[2][...] = db
return dx, dh_prev
def backward(self, dout):
x = self.x
N, T, D = x.shape
W = self.params[0]
dout = dout.reshape(N * T, -1)
rx = x.reshape(N * T, D)
db = np.sum(dout, axis=0)
dW = np.dot(rx.T, dout)
dx = np.dot(dout, W.T)
dx = dx.reshape(N, T, D)
self.grads[0][...] = dW
self.grads[1][...] = db
return dx
def forward(self, x):
N, T, D = x.shape
W, b = self.params
rx = x.reshape(N * T, D)
out = np.dot(rx, W) + b
self.x = x
return out.reshape(N, T, -1)
def cos_similarity(x, y, eps=1e-8):
'''コサイン類似度の算出
:param x: ベクトル
:param y: ベクトル
:param eps: ”0割り”防止のための微小値
:return:
'''
nx = x / (np.sqrt(np.sum(x**2)) + eps)
ny = y / (np.sqrt(np.sum(y**2)) + eps)
return np.dot(nx, ny)
def forward(self, x, h_prev, c_prev):
Wx, Wh, b = self.params
H = h_prev.shape[1]
A = np.dot(x, Wx) + np.dot(h_prev, Wh) + b
# slice
f = A[:, :H]
g = A[:, H:2 * H]
i = A[:, 2 * H:3 * H]
o = A[:, 3 * H:]
f = sigmoid(f)
g = np.tanh(g)
i = sigmoid(i)
o = sigmoid(o)
c_next = f * c_prev + g * i
h_next = o * np.tanh(c_next)
self.cache = (x, h_prev, c_prev, i, f, g, o, c_next)
return h_next, c_next
def forward(self, x):
W, b = self.params
out = np.dot(x, W) + b
self.x = x
return out
def backward(self, dout):
W, = self.params
dx = np.dot(dout, W.T)
dW = np.dot(self.x.T, dout)
self.grads[0][...] = dW
return dx
