# --- Experiment for 3.2 ---
config = netConfig()
config.set(inputD=2, outputD=1, layer=2, nodes=[8, 1], lr=0.03, mode=2, batch=15, \
maxIter=25000 * 4, momentum=0.9, active=active, activeDiff=activeDiff, activeInv=activeInv)
# print (activeInv(config.active(10.)))
net = network(config)
# net.w[0] = np.array([[0.62, 0.47, 1.7, 0.92, -0.70], [1.4, -0.57, 0.042, -1.2, -0.26]])
# net.w[1] = np.array([[-4.7], [-1.0], [4.9], [-4.3], [0.096]])
'''
print(net.forward([5., 5.]))
print(net.values[0])
print(net.values[1])
'''
plt.show()
plt.ion()
data.scatter(x, y, c)
trainProc, testProc = net.backward(Data, c, Data_test, cc)
# data.scatter(x, y, c)
net.draw()
print(net.w)
plt.ioff()
plt.close('all')
data.showTrainProc(2, [trainProc, testProc], ['train', 'test'])
plt.show()
def backward(self, x, y):
assert (self.config.batch <= len(y))
trainProc = []
if self.config.mode == 0: # Delta rule (only for one layer and one output)
for it in range(self.config.maxIter):
delta = [0 for col in range(self.config.inputD)]
randomSamples = random.sample(range(len(x)), self.config.batch)
for b in range(self.config.batch):
index = randomSamples[b]
for i in range(self.config.inputD):
count = 0
for k in range(self.config.inputD):
count = count + self.w[0][k][0] * x[index][k]
count = -self.config.lr * (count -
y[index]) * x[index][i]
delta[i] += count
for i in range(self.config.inputD):
self.w[0][i][0] += delta[i] / self.config.batch
if it % data.CHECK_INTERVAL == 0:
trainProc.append(self.calError(x, y))
if it % (self.config.maxIter / 10) == 0:
data.scatter(np.array(x)[:, 0], np.array(x)[:, 1], y)
self.draw('Delta rule, iter =' + str(it))
plt.close('all')
if self.config.mode == 1: # Perceptron rule (only for one layer and one output)
for it in range(self.config.maxIter):
delta = [0 for col in range(self.config.inputD)]
randomSamples = random.sample(range(len(x)), self.config.batch)
for b in range(self.config.batch):
index = randomSamples[b]
if self.forward(x[index])[0] * y[b] <= 0:
for i in range(self.config.inputD):
delta[i] += y[b] * x[b][i]
for i in range(self.config.inputD):
self.w[0][i][0] += delta[i] / self.config.batch
if it % data.CHECK_INTERVAL == 0:
trainProc.append(self.calError(x, y))
if it % (self.config.maxIter / 10) == 0:
data.scatter(np.array(x)[:, 0], np.array(x)[:, 1], y)
self.draw('Perceptron rule, iter =' + str(it))
plt.close('all')
if self.config.mode == 2: # Back Propogation
for it in range(self.config.maxIter):
delta = []
biasDelta = []
lastDim = self.config.inputD
for i in range(self.config.layer):
delta.append(np.zeros((lastDim, self.config.nodes[i])))
biasDelta.append(np.zeros((self.config.nodes[i], )))
lastDim = self.config.nodes[i]
randomSamples = random.sample(range(len(x)), self.config.batch)
for b in range(self.config.batch):
index = randomSamples[b]
self.forward(x[index])
# initialize delta in the last column
lastDelta = np.zeros((self.config.outputD, ))
lastValue = self.values[self.config.layer - 1]
curInactiveValue = self.inactiveValues[self.config.layer -
1]
for i in range(self.config.outputD):
lastDelta[i] = self.config.activeDiff(
curInactiveValue[i]) * (lastValue[i] - y[index])
# Update layer by layer
for i in range(self.config.layer)[::-1]:
# update bias on layer i
biasDelta[
i] = biasDelta[i] - self.config.lr * lastDelta
# update weights on layer i
curDim = 0
curValue = None
curInactiveValue = None
if i > 0:
curDim = self.config.nodes[i - 1]
curValue = self.values[i - 1]
curInactiveValue = self.inactiveValues[i - 1]
else:
curDim = self.config.inputD
curValue = self.inputData
curInactiveValue = self.inputData
adjustW = -self.config.lr * np.dot(
np.array([curValue]).T, np.array(
[lastDelta])).T.reshape(
(curDim, self.config.nodes[i]))
delta[i] = delta[i] + adjustW
curDelta = np.zeros((curDim, ))
for j in range(curDim):
curDelta[j] = self.config.activeDiff(
curInactiveValue[j])
lastDelta = curDelta * np.dot(
self.w[i],
np.array([lastDelta]).T).reshape((curDim, ))
if it % data.CHECK_INTERVAL == 0:
trainProc.append(self.calLoss(x, y))
if (it + 1) % 2000 == 0:
# #data.scatter2(x, y)
# #self.draw()
self.config.lr = self.config.lr * 0.95
for i in range(self.config.layer):
self.w[i] = self.w[i] + delta[i] / self.config.batch * (
1 - self.config.momentum) + self.lastRoundDelta[
i] / self.config.batch * self.config.momentum
self.bias[i] = self.bias[
i] + biasDelta[i] / self.config.batch * (
1 -
self.config.momentum) + self.lastRoundBiasDelta[
i] / self.config.batch * self.config.momentum
self.lastRoundDelta = delta
self.lastRoundBiasDelta = biasDelta
return trainProc