def forward(self, X, h_old, train=True):
m = self.model
Wz, Wr, Wh, Wy = m['Wz'], m['Wr'], m['Wh'], m['Wy']
bz, br, bh, by = m['bz'], m['br'], m['bh'], m['by']
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
X = np.column_stack((h_old, X_one_hot))
hz, hz_cache = l.fc_forward(X, Wz, bz)
hz, hz_sigm_cache = l.sigmoid_forward(hz)
hr, hr_cache = l.fc_forward(X, Wr, br)
hr, hr_sigm_cache = l.sigmoid_forward(hr)
X_prime = np.column_stack((hr * h_old, X_one_hot))
hh, hh_cache = l.fc_forward(X_prime, Wh, bh)
hh, hh_tanh_cache = l.tanh_forward(hh)
h = (1. - hz) * h_old + hz * hh
y, y_cache = l.fc_forward(h, Wy, by)
cache = (
X, X_prime, h_old, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache,
hh, hh_cache, hh_tanh_cache, h, y_cache
)
if not train:
y = util.softmax(y)
return y, h, cache
def dcross_entropy(y_pred, y_train):
m = y_pred.shape[0]
grad_y = util.softmax(y_pred)
grad_y[range(m), y_train] -= 1.
grad_y /= m
return grad_y
def forward(self, X, h_old, train=True):
m = self.model
Wz, Wr, Wh, Wy = m['Wz'], m['Wr'], m['Wh'], m['Wy']
bz, br, bh, by = m['bz'], m['br'], m['bh'], m['by']
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
X = np.column_stack((h_old, X_one_hot))
hz, hz_cache = l.fc_forward(X, Wz, bz)
hz, hz_sigm_cache = l.sigmoid_forward(hz)
hr, hr_cache = l.fc_forward(X, Wr, br)
hr, hr_sigm_cache = l.sigmoid_forward(hr)
X_prime = np.column_stack((hr * h_old, X_one_hot))
hh, hh_cache = l.fc_forward(X_prime, Wh, bh)
hh, hh_tanh_cache = l.tanh_forward(hh)
h = (1. - hz) * h_old + hz * hh
y, y_cache = l.fc_forward(h, Wy, by)
cache = (
X, X_prime, h_old, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache,
hh, hh_cache, hh_tanh_cache, h, y_cache
)
if not train:
y = util.softmax(y)
return y, h, cache
def dcross_entropy(y_pred, y_train):
m = y_pred.shape[0]
grad_y = util.softmax(y_pred)
grad_y[range(m), y_train] -= 1.
grad_y /= m
return grad_y
def cross_entropy(model, y_pred, y_train, lam=1e-3):
np.set_printoptions(threshold=np.nan, precision=24)
m = y_pred.shape[0]
prob = util.softmax(y_pred)
log_like = -np.log(prob[range(m), y_train])
data_loss = np.sum(log_like) / m
reg_loss = regularization(model, reg_type='l2', lam=lam)
return data_loss + reg_loss
def cross_entropy_noReg(model, y_pred, y_train, lam=0):
m = y_pred.shape[0]
prob = util.softmax(y_pred)
log_like = -np.log(prob[range(m), y_train])
data_loss = np.sum(log_like) / m
return data_loss
def cross_entropy(model, y_pred, y_train, lam=1e-3):
m = y_pred.shape[0]
prob = util.softmax(y_pred)
log_like = -np.log(prob[range(m), y_train])
data_loss = np.sum(log_like) / m
reg_loss = regularization(model, reg_type='l2', lam=lam)
return data_loss + reg_loss
def cross_entropy(model, y_pred, y_train, lam=1e-3):
m = y_pred.shape[0]
prob = util.softmax(y_pred)
log_like = -np.log(prob[range(m), y_train])
data_loss = np.sum(log_like) / m
reg_loss = regularization(model, reg_type='l2', lam=lam)
return data_loss + reg_loss
def forward(self, X, h, train=True):
Wxh, Whh, Why = self.model['Wxh'], self.model['Whh'], self.model['Why']
bh, by = self.model['bh'], self.model['by']
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
hprev = h.copy()
h, h_cache = l.tanh_forward(X_one_hot @ Wxh + hprev @ Whh + bh)
y, y_cache = l.fc_forward(h, Why, by)
cache = (X_one_hot, Whh, h, hprev, y, h_cache, y_cache)
if not train:
y = util.softmax(y)
return y, h, cache
def forward(self, X, h, train=True):
Wxh, Whh, Why = self.model['Wxh'], self.model['Whh'], self.model['Why']
bh, by = self.model['bh'], self.model['by']
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
hprev = h.copy()
h, h_cache = l.tanh_forward(X_one_hot @ Wxh + hprev @ Whh + bh)
y, y_cache = l.fc_forward(h, Why, by)
cache = (X_one_hot, Whh, h, hprev, y, h_cache, y_cache)
if not train:
y = util.softmax(y)
return y, h, cache
def forward(self, X, state, train=True):
m = self.model
Wf, Wi, Wc, Wo, Wy = m['Wf'], m['Wi'], m['Wc'], m['Wo'], m['Wy']
bf, bi, bc, bo, by = m['bf'], m['bi'], m['bc'], m['bo'], m['by']
h_old, c_old = state
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
X = np.column_stack((h_old, X_one_hot))
hf, hf_cache = l.fc_forward(X, Wf, bf)
hf, hf_sigm_cache = l.sigmoid_forward(hf)
hi, hi_cache = l.fc_forward(X, Wi, bi)
hi, hi_sigm_cache = l.sigmoid_forward(hi)
ho, ho_cache = l.fc_forward(X, Wo, bo)
ho, ho_sigm_cache = l.sigmoid_forward(ho)
hc, hc_cache = l.fc_forward(X, Wc, bc)
hc, hc_tanh_cache = l.tanh_forward(hc)
c = hf * c_old + hi * hc
c, c_tanh_cache = l.tanh_forward(c)
h = ho * c
y, y_cache = l.fc_forward(h, Wy, by)
cache = (
X, hf, hi, ho, hc, hf_cache, hf_sigm_cache, hi_cache, hi_sigm_cache, ho_cache,
ho_sigm_cache, hc_cache, hc_tanh_cache, c_old, c, c_tanh_cache, y_cache
)
if not train:
y = util.softmax(y)
return y, (h, c), cache
def forward(self, X, state, train=True):
m = self.model
Wf, Wi, Wc, Wo, Wy = m['Wf'], m['Wi'], m['Wc'], m['Wo'], m['Wy']
bf, bi, bc, bo, by = m['bf'], m['bi'], m['bc'], m['bo'], m['by']
h_old, c_old = state
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
X = np.column_stack((h_old, X_one_hot))
hf, hf_cache = l.fc_forward(X, Wf, bf)
hf, hf_sigm_cache = l.sigmoid_forward(hf)
hi, hi_cache = l.fc_forward(X, Wi, bi)
hi, hi_sigm_cache = l.sigmoid_forward(hi)
ho, ho_cache = l.fc_forward(X, Wo, bo)
ho, ho_sigm_cache = l.sigmoid_forward(ho)
hc, hc_cache = l.fc_forward(X, Wc, bc)
hc, hc_tanh_cache = l.tanh_forward(hc)
c = hf * c_old + hi * hc
c, c_tanh_cache = l.tanh_forward(c)
h = ho * c
y, y_cache = l.fc_forward(h, Wy, by)
cache = (
X, hf, hi, ho, hc, hf_cache, hf_sigm_cache, hi_cache, hi_sigm_cache, ho_cache,
ho_sigm_cache, hc_cache, hc_tanh_cache, c_old, c, c_tanh_cache, y_cache
)
if not train:
y = util.softmax(y)
return y, (h, c), cache
def predict_proba(self, X):
# print(X)
score, _ = self.forward(X, False)
return util.softmax(score)
def predict_proba(self, X):
score, _ = self.forward(X, False)
return util.softmax(score)
def predict_proba(self, X):#not training , but predicting, so train=false
score, _ = self.forward(X, False)
return util.softmax(score)
