def main2(): net = Network(cost="quadratic") net.append(Dense(2, 5)) net.append(Dense(5, 3)) train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]]) train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) sgd = SGD(alpha=0.1, n_epochs=10, mini_batch_size=3) net.fit(train_X, train_Y, sgd)
def main3(): net = Network(cost="categorical_cross_entropy") net.append(Dense(2, 5, activation="sigmoid")) net.append(Dense(5, 3, activation="softmax")) train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]]) train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) sgd = SGD(alpha=0.05, n_epochs=20, mini_batch_size=4, verbosity=1) net.fit(train_X, train_Y, sgd)
def main(): net = Network(cost="quadratic") net.append(Dense(2, 5)) net.append(Dense(5, 3)) # train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]]) # train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) train_X = np.asarray([[0.2, -0.3]]) train_Y = np.asarray([[0.0, 1.0, 0.0]]) print(net.cost(train_X, train_Y)) print(net.cost_gradient(train_X, train_Y))
def main(): # MLP to be fit net_mlp = None with open("/Users/alange/programming/MNIST/store/classic_32.json", "r") as f: net_mlp = Network.read_from_json(f) # Initial particle network net = ParticleNetwork(cost="categorical_cross_entropy", particle_input=ParticleInput(784, s=2.0)) net.append(Particle(784, 32, activation="tanh", zeta=1.0, s=2.0)) net.append(Particle(32, 10, activation="softmax", zeta=1.0, s=2.0)) compute_matrices(net) error = compute_error(net_mlp, net) de_db, de_dq, de_dt, de_drx, de_dry, de_drz = compute_grad_w(net_mlp, net) fd_b, fd_q, fd_t, fd_x, fd_y, fd_z = compute_fd_grad(net_mlp, net) for l, layer in enumerate(net.layers): diff_b = np.mean(de_db[l] - fd_b[l]) diff_q = np.mean(de_dq[l] - fd_q[l]) diff_t = np.mean(de_dt[l] - fd_t[l]) diff_x = np.mean(de_drx[l] - fd_x[l]) print("b", diff_b) print("q", diff_q) print("t", diff_t) print("x", diff_x)
def main(): net = Network(cost="mse") net.append( Convolution2D(input_size=(3, 2), n_filters=1, filter_size=(2, 2), stride=(1, 1), activation="sigmoid")) X = np.asarray([[[0.11, -0.99, 0.222], [-0.30, 0.4, 0.945]], [[0.01, 0.050, 1.005], [-0.22, 0.7, 0.019]], [[0.07, 0.250, -0.56], [-0.24, 0.8, 0.145]], [[0.07, 0.250, -0.56], [-0.24, 0.8, 0.145]]]).reshape( (4, 3, 2)) a = net.feed_to_layer(X, end_layer=0) print(a)
def main(): net = Network(cost="mse") net.append(Convolution1D(input_size=6, filter_size=3, n_filters=1, stride_length=1, activation="sigmoid")) net.append(MaxPool1D(input_size=4, n_filters=1, pool_size=2, stride_length=2)) train_X = np.asarray([[0.2, -0.3, 0.5, 0.5, 0.6, 0.3]]) train_Y = np.asarray([[[0.0, 1.0]]]) sigma = net.predict(train_X) print(sigma) print(net.cost(train_X, train_Y)) db, dw = net.cost_gradient(train_X, train_Y)
def main(): bias_couplings = [ ((0, 0), (1, 1)) ] net = Network(cost="quadratic", regularizer=BiasCoupleL2(coeff_lambda=0.25, couplings=bias_couplings)) net.append(Dense(2, 5)) net.append(Dense(5, 3)) train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]]) train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) # train_X = np.asarray([[0.2, -0.3]]) # train_Y = np.asarray([[0.0, 1.0, 0.0]]) print(net.cost(train_X, train_Y)) print(net.cost_gradient(train_X, train_Y))
def main4(): net = Network(cost="quadratic") net.append(Dense(2, 5)) net.append(Dense(5, 3)) train_X = np.asarray([[0.2, -0.3]]) train_Y = np.asarray([[0.0, 1.0, 0.0]]) with open("/tmp/tmp.json", "w") as f: net.write_to_json(f) net2 = None with open("/tmp/tmp.json", "r") as f: net2 = Network.read_from_json(f) print(net.cost(train_X, train_Y)) print(net2.cost(train_X, train_Y))
def main(): bias_couplings = [((0, 0), (1, 1))] net = Network(cost="quadratic", regularizer=BiasCoupleL2(coeff_lambda=0.25, couplings=bias_couplings)) net.append(Dense(2, 5)) net.append(Dense(5, 3)) train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]]) train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) # train_X = np.asarray([[0.2, -0.3]]) # train_Y = np.asarray([[0.0, 1.0, 0.0]]) print(net.cost(train_X, train_Y)) print(net.cost_gradient(train_X, train_Y))
def fd(): net = Network(cost="categorical_cross_entropy") net.append(Dense(2, 5, activation="tanh")) net.append(Dense(5, 3, activation="linear")) net.append(Dense(3, 3, activation="softmax")) train_X = np.asarray([[0.2, -0.3]]) train_Y = np.asarray([[0.0, 1.0, 0.0]]) train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]]) train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) # Finite difference checking net.cost(train_X, train_Y) db, dw = net.cost_gradient(train_X, train_Y) h = 0.001 print("analytic b") print(db) fd_b = [] for l in range(len(net.layers)): lb = [] for c in range(len(net.layers[l].b)): for b in range(len(net.layers[l].b[c])): orig = net.layers[l].b[c][b] net.layers[l].b[c][b] += h fp = net.cost(train_X, train_Y) net.layers[l].b[c][b] -= 2 * h fm = net.cost(train_X, train_Y) lb.append((fp - fm) / (2 * h)) net.layers[l].b[c][b] = orig fd_b.append(lb) print("numerical b") print(fd_b) print("analytic w") for x in dw: for n in x: print(n) fd_w = [] for l in range(len(net.layers)): lw = [] for w in range(len(net.layers[l].w)): ww = [] for i in range(len(net.layers[l].w[w])): orig = net.layers[l].w[w][i] net.layers[l].w[w][i] += h fp = net.cost(train_X, train_Y) net.layers[l].w[w][i] -= 2 * h fm = net.cost(train_X, train_Y) ww.append((fp - fm) / (2 * h)) net.layers[l].w[w][i] = orig lw.append(ww) fd_w.append(lw) print("numerical w") for x in fd_w: for n in x: print(n)
def fd(): net = Network(cost="categorical_cross_entropy") net.append(Dense(2, 5, activation="tanh")) net.append(Dense(5, 3, activation="linear")) net.append(Dense(3, 3, activation="softmax")) train_X = np.asarray([[0.2, -0.3]]) train_Y = np.asarray([[0.0, 1.0, 0.0]]) train_X = np.asarray([[0.2, -0.3], [0.6, -0.2], [0.8, 0.9], [0.1, 0.1]]) train_Y = np.asarray([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) # Finite difference checking net.cost(train_X, train_Y) db, dw = net.cost_gradient(train_X, train_Y) h = 0.001 print("analytic b") print(db) fd_b = [] for l in range(len(net.layers)): lb = [] for c in range(len(net.layers[l].b)): for b in range(len(net.layers[l].b[c])): orig = net.layers[l].b[c][b] net.layers[l].b[c][b] += h fp = net.cost(train_X, train_Y) net.layers[l].b[c][b] -= 2*h fm = net.cost(train_X, train_Y) lb.append((fp - fm) / (2*h)) net.layers[l].b[c][b] = orig fd_b.append(lb) print("numerical b") print(fd_b) print("analytic w") for x in dw: for n in x: print(n) fd_w = [] for l in range(len(net.layers)): lw = [] for w in range(len(net.layers[l].w)): ww = [] for i in range(len(net.layers[l].w[w])): orig = net.layers[l].w[w][i] net.layers[l].w[w][i] += h fp = net.cost(train_X, train_Y) net.layers[l].w[w][i] -= 2*h fm = net.cost(train_X, train_Y) ww.append((fp - fm) / (2*h)) net.layers[l].w[w][i] = orig lw.append(ww) fd_w.append(lw) print("numerical w") for x in fd_w: for n in x: print(n)
def fd(): net = Network(cost="mse") net.append(Convolution1D(input_size=4, filter_size=2, n_filters=3, stride_length=2, activation="relu", flatten_output=True)) net.append(Dense(6, 2)) # train_X = np.asarray([[0.2, -0.3, 0.5, 0.6, 0.2]]) # train_Y = np.asarray([[[0.0, 0.0, 1.0]]]) # train_Y = np.asarray([[0.0, 0.0, 1.0]]) # train_Y = np.asarray([[[0.0, 0.0, 1.0], [0.5, 0.5, 0.5]]]) # train_Y = np.asarray([[[0.0, 0.0, 1.0, 0.5, 0.5, 0.5]]]) train_X = np.asarray([[0.2, 0.3, 0.4, 0.7]]) train_Y = np.asarray([[0.0, 1.0]]) train_X = np.asarray([[0.2, 0.3, 0.4, 0.7], [0.1, -0.5, -0.9, 0.3]]) train_Y = np.asarray([[0.0, 1.0], [1.0, 0.0]]) # Finite difference checking print(net.predict(train_X)) print(net.cost(train_X, train_Y)) db, dw = net.cost_gradient(train_X, train_Y) h = 0.001 print("analytic b") print(db) fd_b = [] for l in range(len(net.layers)): lb = [] for c in range(len(net.layers[l].b)): for b in range(len(net.layers[l].b[c])): orig = net.layers[l].b[c][b] net.layers[l].b[c][b] += h fp = net.cost(train_X, train_Y) net.layers[l].b[c][b] -= 2*h fm = net.cost(train_X, train_Y) lb.append((fp - fm) / (2*h)) net.layers[l].b[c][b] = orig fd_b.append(lb) print("numerical b") print(fd_b) print("analytic w") for i, x in enumerate(dw): print(i) for n in x: print(n) fd_w = [] for l in range(len(net.layers)): lw = [] for w in range(len(net.layers[l].w)): ww = [] for i in range(len(net.layers[l].w[w])): orig = net.layers[l].w[w][i] net.layers[l].w[w][i] += h fp = net.cost(train_X, train_Y) net.layers[l].w[w][i] -= 2*h fm = net.cost(train_X, train_Y) ww.append((fp - fm) / (2*h)) net.layers[l].w[w][i] = orig lw.append(ww) fd_w.append(lw) print("numerical w") for i, x in enumerate(fd_w): print(i) for n in x: print(n)