def forward(self, X, h_old, train=True):
m = self.model
Wz, Wr, Wh, Wy = m['Wz'], m['Wr'], m['Wh'], m['Wy']
bz, br, bh, by = m['bz'], m['br'], m['bh'], m['by']
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
X = np.column_stack((h_old, X_one_hot))
hz, hz_cache = l.fc_forward(X, Wz, bz)
hz, hz_sigm_cache = l.sigmoid_forward(hz)
hr, hr_cache = l.fc_forward(X, Wr, br)
hr, hr_sigm_cache = l.sigmoid_forward(hr)
X_prime = np.column_stack((hr * h_old, X_one_hot))
hh, hh_cache = l.fc_forward(X_prime, Wh, bh)
hh, hh_tanh_cache = l.tanh_forward(hh)
h = (1. - hz) * h_old + hz * hh
y, y_cache = l.fc_forward(h, Wy, by)
cache = (X, X_prime, h_old, hz, hz_cache, hz_sigm_cache, hr, hr_cache,
hr_sigm_cache, hh, hh_cache, hh_tanh_cache, h, y_cache)
if not train:
y = util.softmax(y)
return y, h, cache
def forward(self, train_data):
self.nodes["A1"] = layer.fc_forward(self.Parameters["W1"], train_data,
self.Parameters["B1"])
self.nodes["Z1"] = activations.relu_forward(self.nodes["A1"])
self.nodes["A2"] = layer.fc_forward(self.Parameters["W2"],
self.nodes["Z1"],
self.Parameters["B2"])
self.nodes["Z2"] = activations.relu_forward(self.nodes["A2"])
self.nodes["A3"] = layer.fc_forward(self.Parameters["W3"],
self.nodes["Z2"],
self.Parameters["B3"])
self.nodes["Z3"] = activations.relu_forward(self.nodes["A3"])
self.nodes["y"] = np.argmax(self.nodes["A3"], axis=0)
return self.nodes["y"]
def forward(self, X, train=False):
gamma1, gamma2 = self.model['gamma1'], self.model['gamma2']
beta1, beta2 = self.model['beta1'], self.model['beta2']
u1, u2 = None, None
bn1_cache, bn2_cache = None, None
# First layer
h1, h1_cache = l.fc_forward(X, self.model['W1'], self.model['b1'])
bn1_cache = (self.bn_caches['bn1_mean'], self.bn_caches['bn1_var'])
h1, bn1_cache, run_mean, run_var = l.bn_forward(h1,
gamma1,
beta1,
bn1_cache,
train=train)
h1, nl_cache1 = self.forward_nonlin(h1)
self.bn_caches['bn1_mean'], self.bn_caches[
'bn1_var'] = run_mean, run_var
if train:
h1, u1 = l.dropout_forward(h1, self.p_dropout)
# Second layer
h2, h2_cache = l.fc_forward(h1, self.model['W2'], self.model['b2'])
bn2_cache = (self.bn_caches['bn2_mean'], self.bn_caches['bn2_var'])
h2, bn2_cache, run_mean, run_var = l.bn_forward(h2,
gamma2,
beta2,
bn2_cache,
train=train)
h2, nl_cache2 = self.forward_nonlin(h2)
self.bn_caches['bn2_mean'], self.bn_caches[
'bn2_var'] = run_mean, run_var
if train:
h2, u2 = l.dropout_forward(h2, self.p_dropout)
# Third layer
score, score_cache = l.fc_forward(h2, self.model['W3'],
self.model['b3'])
cache = (X, h1_cache, h2_cache, score_cache, nl_cache1, nl_cache2, u1,
u2, bn1_cache, bn2_cache)
return score, cache
def forward(self, X, train=False):
# Conv-1
h1, h1_cache = l.conv_forward(X, self.model['W1'], self.model['b1'])
h1, nl_cache1 = l.relu_forward(h1)
# Pool-1
hpool, hpool_cache = l.maxpool_forward(h1)
h2 = hpool.ravel().reshape(X.shape[0], -1)
# FC-7
h3, h3_cache = l.fc_forward(h2, self.model['W2'], self.model['b2'])
h3, nl_cache3 = l.relu_forward(h3)
# Softmax
score, score_cache = l.fc_forward(h3, self.model['W3'], self.model['b3'])
return score, (X, h1_cache, h3_cache, score_cache, hpool_cache, hpool, nl_cache1, nl_cache3)
def forward(self, train_data):
# X1 = train_data[0]
self.nodes["Conv1"] = \
layer.conv_forward(train_data, self.Parameters["K1"], self.Parameters["Kb1"])
self.nodes["Maxpool1"] = layer.max_pooling_forward(self.nodes["Conv1"], (2, 2), (2, 2))
self.nodes["Conv2"] = \
layer.conv_forward(self.nodes["Maxpool1"], self.Parameters["K2"], self.Parameters["Kb2"])
self.nodes["MaxPool2"] = layer.max_pooling_forward(self.nodes["Conv2"], (2, 2))
self.nodes["X2"] = self.nodes["MaxPool2"].reshape((128, -1)).T
self.nodes["A1"] = layer.fc_forward(self.Parameters["W1"], self.nodes["X2"], self.Parameters["B1"])
self.nodes["Z1"] = activations.relu_forward(self.nodes["A1"])
self.nodes["A2"] = layer.fc_forward(self.Parameters["W2"], self.nodes["Z1"], self.Parameters["B2"])
self.nodes["Z2"] = activations.relu_forward(self.nodes["A2"])
self.nodes["A3"] = layer.fc_forward(self.Parameters["W3"], self.nodes["Z2"], self.Parameters["B3"])
self.nodes["Z3"] = activations.relu_forward(self.nodes["A3"])
self.nodes["y"] = np.argmax(self.nodes["A3"], axis=0)
return self.nodes["y"]
def forward(self, X, state, train=True):
m = self.model
Wf, Wi, Wc, Wo, Wy = m['Wf'], m['Wi'], m['Wc'], m['Wo'], m['Wy']
bf, bi, bc, bo, by = m['bf'], m['bi'], m['bc'], m['bo'], m['by']
h_old, c_old = state
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
X = np.column_stack((h_old, X_one_hot))
hf, hf_cache = l.fc_forward(X, Wf, bf)
hf, hf_sigm_cache = l.sigmoid_forward(hf)
hi, hi_cache = l.fc_forward(X, Wi, bi)
hi, hi_sigm_cache = l.sigmoid_forward(hi)
ho, ho_cache = l.fc_forward(X, Wo, bo)
ho, ho_sigm_cache = l.sigmoid_forward(ho)
hc, hc_cache = l.fc_forward(X, Wc, bc)
hc, hc_tanh_cache = l.tanh_forward(hc)
c = hf * c_old + hi * hc
c, c_tanh_cache = l.tanh_forward(c)
h = ho * c
y, y_cache = l.fc_forward(h, Wy, by)
cache = (X, hf, hi, ho, hc, hf_cache, hf_sigm_cache, hi_cache,
hi_sigm_cache, ho_cache, ho_sigm_cache, hc_cache,
hc_tanh_cache, c_old, c, c_tanh_cache, y_cache)
if not train:
y = util.softmax(y)
return y, (h, c), cache
def forward(self, X, h, train=True):
Wxh, Whh, Why = self.model['Wxh'], self.model['Whh'], self.model['Why']
bh, by = self.model['bh'], self.model['by']
X_one_hot = np.zeros(self.D)
X_one_hot[X] = 1.
X_one_hot = X_one_hot.reshape(1, -1)
hprev = h.copy()
h, h_cache = l.tanh_forward(X_one_hot @ Wxh + hprev @ Whh + bh)
y, y_cache = l.fc_forward(h, Why, by)
cache = (X_one_hot, Whh, h, hprev, y, h_cache, y_cache)
if not train:
y = util.softmax(y)
return y, h, cache