def main2():
net = Network(cost="quadratic")
net.append(Dense(2, 5))
net.append(Dense(5, 3))
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
sgd = SGD(alpha=0.1, n_epochs=10, mini_batch_size=3)
net.fit(train_X, train_Y, sgd)
def main3():
net = Network(cost="categorical_cross_entropy")
net.append(Dense(2, 5, activation="sigmoid"))
net.append(Dense(5, 3, activation="softmax"))
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
sgd = SGD(alpha=0.05, n_epochs=20, mini_batch_size=4, verbosity=1)
net.fit(train_X, train_Y, sgd)
def main():
net = Network(cost="quadratic")
net.append(Dense(2, 5))
net.append(Dense(5, 3))
# train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
# train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
train_X = np.asarray([[0.2, -0.3]])
train_Y = np.asarray([[0.0, 1.0, 0.0]])
print(net.cost(train_X, train_Y))
print(net.cost_gradient(train_X, train_Y))
def main():
# MLP to be fit
net_mlp = None
with open("/Users/alange/programming/MNIST/store/classic_32.json",
"r") as f:
net_mlp = Network.read_from_json(f)
# Initial particle network
net = ParticleNetwork(cost="categorical_cross_entropy",
particle_input=ParticleInput(784, s=2.0))
net.append(Particle(784, 32, activation="tanh", zeta=1.0, s=2.0))
net.append(Particle(32, 10, activation="softmax", zeta=1.0, s=2.0))
compute_matrices(net)
error = compute_error(net_mlp, net)
de_db, de_dq, de_dt, de_drx, de_dry, de_drz = compute_grad_w(net_mlp, net)
fd_b, fd_q, fd_t, fd_x, fd_y, fd_z = compute_fd_grad(net_mlp, net)
for l, layer in enumerate(net.layers):
diff_b = np.mean(de_db[l] - fd_b[l])
diff_q = np.mean(de_dq[l] - fd_q[l])
diff_t = np.mean(de_dt[l] - fd_t[l])
diff_x = np.mean(de_drx[l] - fd_x[l])
print("b", diff_b)
print("q", diff_q)
print("t", diff_t)
print("x", diff_x)
def main():
net = Network(cost="mse")
net.append(
Convolution2D(input_size=(3, 2),
n_filters=1,
filter_size=(2, 2),
stride=(1, 1),
activation="sigmoid"))
X = np.asarray([[[0.11, -0.99, 0.222], [-0.30, 0.4, 0.945]],
[[0.01, 0.050, 1.005], [-0.22, 0.7, 0.019]],
[[0.07, 0.250, -0.56], [-0.24, 0.8, 0.145]],
[[0.07, 0.250, -0.56], [-0.24, 0.8, 0.145]]]).reshape(
(4, 3, 2))
a = net.feed_to_layer(X, end_layer=0)
print(a)
def main():
net = Network(cost="mse")
net.append(Convolution1D(input_size=6, filter_size=3, n_filters=1, stride_length=1,
activation="sigmoid"))
net.append(MaxPool1D(input_size=4, n_filters=1, pool_size=2, stride_length=2))
train_X = np.asarray([[0.2, -0.3, 0.5, 0.5, 0.6, 0.3]])
train_Y = np.asarray([[[0.0, 1.0]]])
sigma = net.predict(train_X)
print(sigma)
print(net.cost(train_X, train_Y))
db, dw = net.cost_gradient(train_X, train_Y)
def main():
bias_couplings = [
((0, 0), (1, 1))
]
net = Network(cost="quadratic", regularizer=BiasCoupleL2(coeff_lambda=0.25, couplings=bias_couplings))
net.append(Dense(2, 5))
net.append(Dense(5, 3))
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
# train_X = np.asarray([[0.2, -0.3]])
# train_Y = np.asarray([[0.0, 1.0, 0.0]])
print(net.cost(train_X, train_Y))
print(net.cost_gradient(train_X, train_Y))
def main4():
net = Network(cost="quadratic")
net.append(Dense(2, 5))
net.append(Dense(5, 3))
train_X = np.asarray([[0.2, -0.3]])
train_Y = np.asarray([[0.0, 1.0, 0.0]])
with open("/tmp/tmp.json", "w") as f:
net.write_to_json(f)
net2 = None
with open("/tmp/tmp.json", "r") as f:
net2 = Network.read_from_json(f)
print(net.cost(train_X, train_Y))
print(net2.cost(train_X, train_Y))
def main3():
net = Network(cost="categorical_cross_entropy")
net.append(Dense(2, 5, activation="sigmoid"))
net.append(Dense(5, 3, activation="softmax"))
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0],
[1.0, 0.0, 0.0]])
sgd = SGD(alpha=0.05, n_epochs=20, mini_batch_size=4, verbosity=1)
net.fit(train_X, train_Y, sgd)
def main2():
net = Network(cost="quadratic")
net.append(Dense(2, 5))
net.append(Dense(5, 3))
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0],
[1.0, 0.0, 0.0]])
sgd = SGD(alpha=0.1, n_epochs=10, mini_batch_size=3)
net.fit(train_X, train_Y, sgd)
def main():
net = Network(cost="quadratic")
net.append(Dense(2, 5))
net.append(Dense(5, 3))
# train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
# train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
train_X = np.asarray([[0.2, -0.3]])
train_Y = np.asarray([[0.0, 1.0, 0.0]])
print(net.cost(train_X, train_Y))
print(net.cost_gradient(train_X, train_Y))
def main4():
net = Network(cost="quadratic")
net.append(Dense(2, 5))
net.append(Dense(5, 3))
train_X = np.asarray([[0.2, -0.3]])
train_Y = np.asarray([[0.0, 1.0, 0.0]])
with open("/tmp/tmp.json", "w") as f:
net.write_to_json(f)
net2 = None
with open("/tmp/tmp.json", "r") as f:
net2 = Network.read_from_json(f)
print(net.cost(train_X, train_Y))
print(net2.cost(train_X, train_Y))
def main():
bias_couplings = [((0, 0), (1, 1))]
net = Network(cost="quadratic",
regularizer=BiasCoupleL2(coeff_lambda=0.25,
couplings=bias_couplings))
net.append(Dense(2, 5))
net.append(Dense(5, 3))
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0],
[1.0, 0.0, 0.0]])
# train_X = np.asarray([[0.2, -0.3]])
# train_Y = np.asarray([[0.0, 1.0, 0.0]])
print(net.cost(train_X, train_Y))
print(net.cost_gradient(train_X, train_Y))
def fd():
net = Network(cost="categorical_cross_entropy")
net.append(Dense(2, 5, activation="tanh"))
net.append(Dense(5, 3, activation="linear"))
net.append(Dense(3, 3, activation="softmax"))
train_X = np.asarray([[0.2, -0.3]])
train_Y = np.asarray([[0.0, 1.0, 0.0]])
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0],
[1.0, 0.0, 0.0]])
# Finite difference checking
net.cost(train_X, train_Y)
db, dw = net.cost_gradient(train_X, train_Y)
h = 0.001
print("analytic b")
print(db)
fd_b = []
for l in range(len(net.layers)):
lb = []
for c in range(len(net.layers[l].b)):
for b in range(len(net.layers[l].b[c])):
orig = net.layers[l].b[c][b]
net.layers[l].b[c][b] += h
fp = net.cost(train_X, train_Y)
net.layers[l].b[c][b] -= 2 * h
fm = net.cost(train_X, train_Y)
lb.append((fp - fm) / (2 * h))
net.layers[l].b[c][b] = orig
fd_b.append(lb)
print("numerical b")
print(fd_b)
print("analytic w")
for x in dw:
for n in x:
print(n)
fd_w = []
for l in range(len(net.layers)):
lw = []
for w in range(len(net.layers[l].w)):
ww = []
for i in range(len(net.layers[l].w[w])):
orig = net.layers[l].w[w][i]
net.layers[l].w[w][i] += h
fp = net.cost(train_X, train_Y)
net.layers[l].w[w][i] -= 2 * h
fm = net.cost(train_X, train_Y)
ww.append((fp - fm) / (2 * h))
net.layers[l].w[w][i] = orig
lw.append(ww)
fd_w.append(lw)
print("numerical w")
for x in fd_w:
for n in x:
print(n)
def fd():
net = Network(cost="categorical_cross_entropy")
net.append(Dense(2, 5, activation="tanh"))
net.append(Dense(5, 3, activation="linear"))
net.append(Dense(3, 3, activation="softmax"))
train_X = np.asarray([[0.2, -0.3]])
train_Y = np.asarray([[0.0, 1.0, 0.0]])
train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]])
train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
# Finite difference checking
net.cost(train_X, train_Y)
db, dw = net.cost_gradient(train_X, train_Y)
h = 0.001
print("analytic b")
print(db)
fd_b = []
for l in range(len(net.layers)):
lb = []
for c in range(len(net.layers[l].b)):
for b in range(len(net.layers[l].b[c])):
orig = net.layers[l].b[c][b]
net.layers[l].b[c][b] += h
fp = net.cost(train_X, train_Y)
net.layers[l].b[c][b] -= 2*h
fm = net.cost(train_X, train_Y)
lb.append((fp - fm) / (2*h))
net.layers[l].b[c][b] = orig
fd_b.append(lb)
print("numerical b")
print(fd_b)
print("analytic w")
for x in dw:
for n in x:
print(n)
fd_w = []
for l in range(len(net.layers)):
lw = []
for w in range(len(net.layers[l].w)):
ww = []
for i in range(len(net.layers[l].w[w])):
orig = net.layers[l].w[w][i]
net.layers[l].w[w][i] += h
fp = net.cost(train_X, train_Y)
net.layers[l].w[w][i] -= 2*h
fm = net.cost(train_X, train_Y)
ww.append((fp - fm) / (2*h))
net.layers[l].w[w][i] = orig
lw.append(ww)
fd_w.append(lw)
print("numerical w")
for x in fd_w:
for n in x:
print(n)
def fd():
net = Network(cost="mse")
net.append(Convolution1D(input_size=4, filter_size=2, n_filters=3, stride_length=2,
activation="relu", flatten_output=True))
net.append(Dense(6, 2))
# train_X = np.asarray([[0.2, -0.3, 0.5, 0.6, 0.2]])
# train_Y = np.asarray([[[0.0, 0.0, 1.0]]])
# train_Y = np.asarray([[0.0, 0.0, 1.0]])
# train_Y = np.asarray([[[0.0, 0.0, 1.0], [0.5, 0.5, 0.5]]])
# train_Y = np.asarray([[[0.0, 0.0, 1.0, 0.5, 0.5, 0.5]]])
train_X = np.asarray([[0.2, 0.3, 0.4, 0.7]])
train_Y = np.asarray([[0.0, 1.0]])
train_X = np.asarray([[0.2, 0.3, 0.4, 0.7], [0.1, -0.5, -0.9, 0.3]])
train_Y = np.asarray([[0.0, 1.0], [1.0, 0.0]])
# Finite difference checking
print(net.predict(train_X))
print(net.cost(train_X, train_Y))
db, dw = net.cost_gradient(train_X, train_Y)
h = 0.001
print("analytic b")
print(db)
fd_b = []
for l in range(len(net.layers)):
lb = []
for c in range(len(net.layers[l].b)):
for b in range(len(net.layers[l].b[c])):
orig = net.layers[l].b[c][b]
net.layers[l].b[c][b] += h
fp = net.cost(train_X, train_Y)
net.layers[l].b[c][b] -= 2*h
fm = net.cost(train_X, train_Y)
lb.append((fp - fm) / (2*h))
net.layers[l].b[c][b] = orig
fd_b.append(lb)
print("numerical b")
print(fd_b)
print("analytic w")
for i, x in enumerate(dw):
print(i)
for n in x:
print(n)
fd_w = []
for l in range(len(net.layers)):
lw = []
for w in range(len(net.layers[l].w)):
ww = []
for i in range(len(net.layers[l].w[w])):
orig = net.layers[l].w[w][i]
net.layers[l].w[w][i] += h
fp = net.cost(train_X, train_Y)
net.layers[l].w[w][i] -= 2*h
fm = net.cost(train_X, train_Y)
ww.append((fp - fm) / (2*h))
net.layers[l].w[w][i] = orig
lw.append(ww)
fd_w.append(lw)
print("numerical w")
for i, x in enumerate(fd_w):
print(i)
for n in x:
print(n)
