def train_step(self, X_train, y_train, state): y_preds = [] caches = [] loss = 0. # Forward for x, y_true in zip(X_train, y_train): y, state, cache = self.forward(x, state, train=True) loss += loss_fun.cross_entropy(self.model, y, y_true, lam=0) y_preds.append(y) caches.append(cache) loss /= X_train.shape[0] # Backward dh_next = np.zeros((1, self.H)) dc_next = np.zeros((1, self.H)) d_next = (dh_next, dc_next) grads = {k: np.zeros_like(v) for k, v in self.model.items()} for y_pred, y_true, cache in reversed(list(zip(y_preds, y_train, caches))): grad, d_next = self.backward(y_pred, y_true, d_next, cache) for k in grads.keys(): grads[k] += grad[k] for k, v in grads.items(): grads[k] = np.clip(v, -5., 5.) return grads, loss, state
def train_step(self, X_train, y_train, state): y_preds = [] caches = [] loss = 0. # Forward for x, y_true in zip(X_train, y_train): y, state, cache = self.forward(x, state, train=True) loss += loss_fun.cross_entropy(self.model, y, y_true, lam=0) y_preds.append(y) caches.append(cache) loss /= X_train.shape[0] # Backward dh_next = np.zeros((1, self.H)) dc_next = np.zeros((1, self.H)) d_next = (dh_next, dc_next) grads = {k: np.zeros_like(v) for k, v in self.model.items()} for y_pred, y_true, cache in reversed(list(zip(y_preds, y_train, caches))): grad, d_next = self.backward(y_pred, y_true, d_next, cache) for k in grads.keys(): grads[k] += grad[k] for k, v in grads.items(): grads[k] = np.clip(v, -5., 5.) return grads, loss, state
def train_step(self, X_train, y_train, h): ys = [] caches = [] loss = 0. # Forward for x, y in zip(X_train, y_train): y_pred, h, cache = self.forward(x, h, train=True) loss += loss_fun.cross_entropy(self.model, y_pred, y, lam=0) ys.append(y_pred) caches.append(cache) loss /= X_train.shape[0] # Backward dh_next = np.zeros((1, self.H)) grads = {k: np.zeros_like(v) for k, v in self.model.items()} for t in reversed(range(len(X_train))): grad, dh_next = self.backward(ys[t], y_train[t], dh_next, caches[t]) for k in grads.keys(): grads[k] += grad[k] for k, v in grads.items(): grads[k] = np.clip(v, -5., 5.) return grads, loss, h
def train_step(self, X_train, y_train, h): ys = [] caches = [] loss = 0. # Forward for x, y in zip(X_train, y_train): y_pred, h, cache = self.forward(x, h, train=True) loss += loss_fun.cross_entropy(self.model, y_pred, y, lam=0) ys.append(y_pred) caches.append(cache) loss /= X_train.shape[0] # Backward dh_next = np.zeros((1, self.H)) grads = {k: np.zeros_like(v) for k, v in self.model.items()} for t in reversed(range(len(X_train))): grad, dh_next = self.backward(ys[t], y_train[t], dh_next, caches[t]) for k in grads.keys(): grads[k] += grad[k] for k, v in grads.items(): grads[k] = np.clip(v, -5., 5.) return grads, loss, h