def train_step(self, X_train, y_train, state):
y_preds = []
caches = []
loss = 0.
# Forward
for x, y_true in zip(X_train, y_train):
y, state, cache = self.forward(x, state, train=True)
loss += loss_fun.cross_entropy(self.model, y, y_true, lam=0)
y_preds.append(y)
caches.append(cache)
loss /= X_train.shape[0]
# Backward
dh_next = np.zeros((1, self.H))
dc_next = np.zeros((1, self.H))
d_next = (dh_next, dc_next)
grads = {k: np.zeros_like(v) for k, v in self.model.items()}
for y_pred, y_true, cache in reversed(list(zip(y_preds, y_train, caches))):
grad, d_next = self.backward(y_pred, y_true, d_next, cache)
for k in grads.keys():
grads[k] += grad[k]
for k, v in grads.items():
grads[k] = np.clip(v, -5., 5.)
return grads, loss, state
def train_step(self, X_train, y_train, state):
y_preds = []
caches = []
loss = 0.
# Forward
for x, y_true in zip(X_train, y_train):
y, state, cache = self.forward(x, state, train=True)
loss += loss_fun.cross_entropy(self.model, y, y_true, lam=0)
y_preds.append(y)
caches.append(cache)
loss /= X_train.shape[0]
# Backward
dh_next = np.zeros((1, self.H))
dc_next = np.zeros((1, self.H))
d_next = (dh_next, dc_next)
grads = {k: np.zeros_like(v) for k, v in self.model.items()}
for y_pred, y_true, cache in reversed(list(zip(y_preds, y_train, caches))):
grad, d_next = self.backward(y_pred, y_true, d_next, cache)
for k in grads.keys():
grads[k] += grad[k]
for k, v in grads.items():
grads[k] = np.clip(v, -5., 5.)
return grads, loss, state
def train_step(self, X_train, y_train, h):
ys = []
caches = []
loss = 0.
# Forward
for x, y in zip(X_train, y_train):
y_pred, h, cache = self.forward(x, h, train=True)
loss += loss_fun.cross_entropy(self.model, y_pred, y, lam=0)
ys.append(y_pred)
caches.append(cache)
loss /= X_train.shape[0]
# Backward
dh_next = np.zeros((1, self.H))
grads = {k: np.zeros_like(v) for k, v in self.model.items()}
for t in reversed(range(len(X_train))):
grad, dh_next = self.backward(ys[t], y_train[t], dh_next, caches[t])
for k in grads.keys():
grads[k] += grad[k]
for k, v in grads.items():
grads[k] = np.clip(v, -5., 5.)
return grads, loss, h
def train_step(self, X_train, y_train, h):
ys = []
caches = []
loss = 0.
# Forward
for x, y in zip(X_train, y_train):
y_pred, h, cache = self.forward(x, h, train=True)
loss += loss_fun.cross_entropy(self.model, y_pred, y, lam=0)
ys.append(y_pred)
caches.append(cache)
loss /= X_train.shape[0]
# Backward
dh_next = np.zeros((1, self.H))
grads = {k: np.zeros_like(v) for k, v in self.model.items()}
for t in reversed(range(len(X_train))):
grad, dh_next = self.backward(ys[t], y_train[t], dh_next, caches[t])
for k in grads.keys():
grads[k] += grad[k]
for k, v in grads.items():
grads[k] = np.clip(v, -5., 5.)
return grads, loss, h