def LM_batch(type, X, yr, p, win, re_learn, epoch1, epoch2, learning_rate): start = time.time() L = int(yr.shape[0]) X = nn.input_shape(X) yr = nn.output_shape(yr) """The dimension of neuron and all matrixes like weights and so on depends on number of columns of input vector""" DIM = nn.input_dim(X[0, :]) """Creating Neuron object with particular dimension""" if type == 'CNU': XNU_B = nn.HONU().CNU_Batch(DIM) # +number_of_rec) col_W = XNU_B.col_W_from_W(XNU_B.W_CNU_init) if type == 'QNU': XNU_B = nn.HONU().QNU_Batch(DIM) # +number_of_rec) col_W = XNU_B.col_W_from_W(XNU_B.W_QNU_init) if type == 'LNU': XNU_B = nn.HONU().LNU_Batch(DIM) # +number_of_rec) col_W = XNU_B.col_W_from_W(XNU_B.W_LNU_init) """Creating Method object(BackPropagation) with particular method""" BP = nn.Backpropagation() yn = np.zeros(L + p) yn_temporary = np.zeros(L + p) JAC = XNU_B.Jacobian(X[:, :]) #J = XNU_B.Jacobian(X[:win,:]) J = JAC[:win, :] print(col_W.shape, J.shape) for i in range(epoch1): yn[p:p + win] = XNU_B.Y(J, col_W) col_W, dw = BP.LM( yr[p:p + win], yn[p:p + win], learning_rate, col_W, J) # y_target,y_neuron, learning rate, col_W, Jacobian(col_X) """Sliding window""" for k in range(win + p, L): """Re-Learn every x samples, here we relearn every day""" if k % (re_learn) == 0 and k >= win + p: # Retrain J = JAC[k - win - p + 1:k - p + 1, :] print(col_W.shape, J.shape) for epoch in range(epoch2): yn_temporary[k - win + 1:k + 1] = XNU_B.Y( J, col_W ) # u LM je potreba spocitat i vystup z neuronu - proto je algoritmus pomaly oproti CGD col_W, dw = BP.LM(yr[k - win + 1:k + 1], yn_temporary[k - win + 1:k + 1], learning_rate, col_W, J) print('Finished: ' + str(round((100.0 * ((k * 1.0 - win + 1)) / (L + 1 - win)), 2)) + '%') """For prediction we only need one row of samples as they include all reccurent values""" #XX = X[k :k+1, :] # staci X[k-1,:], takto je temporary.shape = 1, #J = XNU_B.Jacobian(XX) J = JAC[k:k + 1, :] temporary = XNU_B.Y(J, col_W) yn[k + p] = temporary[-1] return yn
def XGD_sample_MLP(type, nodes, X, yr, p, win, re_learn, epoch1, epoch2, learning_rate): # NGD = 0.005, GD = NGD/10 L = int(yr.shape[0]) X = nn.input_shape(X) yr = nn.output_shape(yr) """The dimension of neuron and all matrixes like weights and so on depends on number of columns of input vector""" X = nn.X_bias(X) DIM = nn.input_dim(X[0, :]) """Creating Neuron object with particular dimension""" mlp = nn.MLP(DIM, nodes) W = mlp.W V = mlp.V """Creating Method object(BackPropagation) with particular method""" BP = nn.Backpropagation() SSE = 0 yn = np.zeros(L + p) e = np.zeros(win) SSE = 0 for epoch in range(epoch1): Jww, Jv, e, e_new = mlp.Jw_Jv(W, V, X[:win], yr[p:p + win], win) # spocitani for i in range(win): V = BP.NGD_MLP(e[i], learning_rate, Jv[i, :], V) # dv = np.dot(np.dot(np.linalg.inv((np.dot(Jv.T, Jv) + 1. / muv * Lv)), Jv.T), e) # V = V + dv for nod in range(nodes): Jw = Jww[:, nod, :] W[nod, :] = BP.NGD_MLP(e[i], learning_rate, Jw[i, :], W[nod, :]) SSE = np.append(SSE, np.dot(e, e)) """Sliding window""" for k in range(win + p, L): """Re-Learn every x samples, here we relearn every day""" if k % (re_learn) == 0 and k >= win + p: # Retrain # XX = X[k - win-p+1:k-p+1, :] # J = XNU_B.Jacobian(XX) X_ = X[k - win - p + 1:k - p + 1, :] Jww, Jv, e, e_new = mlp.Jw_Jv(W, V, X_, yr[k - win + 1:k + 1], win) for epoch in range(epoch2): Jww, Jv, e, e_new = mlp.Jw_Jv(W, V, X_, yr[k - win + 1:k + 1], win) for i in range(win): if type == 'NGD': V = BP.NGD_MLP(e[i], learning_rate, Jv[i, :], V) if type == 'GD': V = BP.GD_MLP(e[i], learning_rate, Jv[i, :], V) # dv = np.dot(np.dot(np.linalg.inv((np.dot(Jv.T, Jv) + 1. / muv * Lv)), Jv.T), e) # V = V + dv for nod in range(nodes): Jw = Jww[:, nod, :] if type == 'NGD': W[nod, :] = BP.NGD_MLP(e[i], learning_rate, Jw[i, :], W[nod, :]) if type == 'GD': W[nod, :] = BP.GD_MLP(e[i], learning_rate, Jw[i, :], W[nod, :]) SSE = np.append(SSE, np.dot(e, e)) print('Finished: ' + str(round((100.0 * ((k * 1.0 - win + 1)) / (L + 1 - win)), 2)) + '%') """For prediction we only need one row of samples as they include all reccurent values""" Nu = np.dot(W, X[k, :].T) # n1 x N X1 = mlp.phi(Nu) yn[k + p] = np.dot(V, X1) # pass return yn, SSE
def batch_MLP(method, nodes, X, yr, p, win, re_learn, epoch1, epoch2, learning_rate): L = int(yr.shape[0]) X = nn.input_shape(X) yr = nn.output_shape(yr) """The dimension of neuron and all matrixes like weights and so on depends on number of columns of input vector""" X = nn.X_bias(X) DIM = nn.input_dim(X[0, :]) """Creating Neuron object with particular dimension""" mlp = nn.MLP(DIM, nodes) W = mlp.W V = mlp.V BP = nn.Backpropagation() # CGD u MLP NEFUNGUJE CGD = BP.CGD() """Creating Method object(BackPropagation) with particular method""" yn = np.zeros(L + p) VV = np.zeros((L + p, nodes)) SSE = 0 if method == 'LM': for i in range( epoch1): # LM algoritmus, je potreba prepocitavat Jakobian J, e = mlp.Jacobian( W, V, X[:win, :], yr[p:p + win], win) # Spocita Jakobianu a chyby, viz neural_network.py W, V = mlp.W_V_LM(W, V, J, learning_rate, e) # nauceni SSEE = np.dot(e, e) SSE = np.append(SSE, SSEE) # Nefunguje - a ani pry nemuze fungovat if method == 'CGD': J, e = mlp.Jacobian(W, V, X[:win, :], yr[p:p + win], win) A = CGD.A(J) b = CGD.b(yr[p:p + win], J) col_W = mlp.pack_WV(W, V) re = CGD.re(b, A, col_W) pp = re for epoch in range(epoch2): J, e = mlp.Jacobian(W, V, X[:win, :], yr[p:p + win], win) A = CGD.A(J) col_W = mlp.pack_WV(W, V) col_W, pp, re = CGD.CGD(A, col_W, re, pp) W, V = mlp.unpack_WV(col_W) """Sliding window""" for k in range(win + p, L): """Re-Learn every x samples, here we relearn every day""" if k % (re_learn) == 0 and k >= win + p: # Retrain print('Finished: ' + str(round((100.0 * ((k * 1.0 - win + 1)) / (L + 1 - win)), 2)) + '%') J, e = mlp.Jacobian(W, V, X[k - win - p + 1:k - p + 1, :], yr[k - win + 1:k + 1], win) A = CGD.A(J) # CGD nefunguje b = CGD.b(yr[p:p + win], J) # CGD nefunguje col_W = mlp.pack_WV(W, V) # CGD nefunguje re = CGD.re(b, A, col_W) # CGD nefunguje pp = re # CGD nefunguje for i in range(epoch2): if method == 'LM': J, e = mlp.Jacobian(W, V, X[k - win - p + 1:k - p + 1, :], yr[k - win + 1:k + 1], win) W, V = mlp.W_V_LM(W, V, J, learning_rate, e) print('Finished: ' + str( round((100.0 * ((k * 1.0 - win + 1)) / (L + 1 - win)), 2)) + '%') if method == 'CGD': J, e = mlp.Jacobian(W, V, X[k - win - p + 1:k - p + 1, :], yr[k - win + 1:k + 1], win) A = CGD.A(J) col_W, pp, re = CGD.CGD(A, col_W, re, pp) W, V = mlp.unpack_WV(col_W) SSEE = np.dot(abs(e), abs(e)) SSE = np.append(SSE, SSEE) VV[k, :] = V # ? v = np.dot(W, X[k:k + 1, :].T) # n1 x N phi = mlp.phi(v) y_temp = mlp.Y(V, phi) yn[k + p] = y_temp[-1] return yn, SSE
def NGD_sample_MLP(nodes, X, yr, p, win, re_learn, epoch1, epoch2, learning_rate): L = int(yr.shape[0]) X = nn.input_shape(X) yr = nn.output_shape(yr) """The dimension of neuron and all matrixes like weights and so on depends on number of columns of input vector""" #POZOR u mlp se pridava bias manualne -> Nikoliv vsak v aplikaci X = nn.X_bias(X) DIM = nn.input_dim(X[0, :]) # rozmer vstupni matice i s Biasem """Creating Neuron object with particular dimension""" mlp = nn.MLP(DIM, nodes) # nodes = pocet neuronu ve skryte vrstve W = mlp.W V = mlp.V yn = np.zeros(L + p) e = np.zeros(win) print(X.shape) for epoch in range(epoch1): for k in range(win): # klouzajici okno x = X[k, :] # vstupni vektor ze vstupni matice nu = np.dot(W, x) # agregace x1 = mlp.phi(nu) # 1st layer output - vystup ze skryte vrstvy yn[k + p] = np.dot(V, x1) # vystup ze site e[k + p] = yr[k + p] - yn[k + p] # chyba # hidden layer updates for i in range(nodes): # uceni jednotlivych neuronu skryte vrstvy dyndWi = V[i] * mlp.dphidv(nu[i]) * x dWi = learning_rate / (1 + np.sum(x**2)) * e[k + p] * dyndWi W[i, :] = W[i, :] + dWi # output layer updates # uceni vystupniho neuronu dyndv = x1 dv = learning_rate * e[k + p] * dyndv V = V + dv #SSE[epoch] = sum((e[ny + p - 1:] * 3 * self.stdy) ** 2) """Sliding window""" for k in range(win + p, L): """Re-Learn every x samples, here we relearn every day""" if k % (re_learn) == 0 and k >= win + p: # Retrain X_ = X[ k - win - p + 1:k - p + 1, :] # kratkodoba vstupni matice pro dany usek pretrenovani e = np.zeros(win) for epoch in range(epoch2): for ii in range( win ): # klouzajici okno - stejny princip jako pri predtrenovani x = X_[ii, :] nu = np.dot(W, x) x1 = mlp.phi(nu) # 1st layer output yn[k - win + 1 + ii] = np.sum(V * x1) yn_temp = np.sum(V * x1) #e[k -win +1+ii] = yr[k -win +1+ii] - yn[k -win +1+ii] e[ii] = yr[k - win + 1 + ii] - yn_temp # hidden layer updates for i in range(nodes): dyndWi = V[i] * mlp.dphidv(nu[i]) * x #dWi = learning_rate * e_temp * dyndWi dWi = learning_rate / (1 + np.sum(x**2)) * e[ii] * dyndWi W[i, :] = W[i, :] + dWi dyndv = x1 dv = learning_rate * e[ii] * dyndv V = V + dv print('Finished: ' + str(round((100.0 * ((k * 1.0 - win + 1)) / (L + 1 - win)), 2)) + '%') """For prediction we only need one row of samples as they include all reccurent values""" # XX = X[k :k+1, :] # staci X[k-1,:], takto je temporary.shape = 1, # J = XNU_B.Jacobian(XX) x = X[k, :] nu = np.dot(W, x) x1 = mlp.phi(nu) # 1st layer output yn[k + p] = np.sum(V * x1) #try: # print(yr[k+p] - yn[k+p]) #except: # pass return yn
def NGD_sample_HONU(type, X, yr, p, win, re_learn, epoch1, epoch2, learning_rate): start = time.time() L = int(yr.shape[0]) X = nn.input_shape(X) yr = nn.output_shape(yr) """The dimension of neuron and all matrixes like weights and so on depends on number of columns of input vector""" DIM = nn.input_dim(X[0, :]) """Creating Neuron object with particular dimension""" if type == 'CNU': XNU_S = nn.HONU().CNU_Batch(DIM) # +number_of_rec) col_W = XNU_S.col_W_from_W(XNU_S.W_CNU_init) if type == 'QNU': XNU_S = nn.HONU().QNU_Batch(DIM) # +number_of_rec) col_W = XNU_S.col_W_from_W(XNU_S.W_QNU_init) if type == 'LNU': XNU_S = nn.HONU().LNU_Batch(DIM) # +number_of_rec) col_W = XNU_S.col_W_from_W(XNU_S.W_LNU_init) """Creating Method object(BackPropagation) with particular method""" BP = nn.Backpropagation() yn = np.zeros(L + p) e = np.zeros(L + p) JAC = XNU_S.Jacobian(X[:, :]) # J = XNU_B.Jacobian(X[:win,:]) J = JAC[:win, :] SSE = 0 for epoch in range(epoch1): # Epoch trenovani for k in range(win): # Klouzajici okno j = J[k, :] yn[k + p] = XNU_S.Y(j, col_W) e[k + p] = yr[k + p] - yn[k + p] col_W = BP.NGD(yr[k + p], yn[k + p], learning_rate, j, col_W) SSE = np.append(SSE, np.sum((e[p:p + win])**2)) #for i in range(epoch1): # yn[p:p + win] = XNU_B.Y(J, col_W) # col_W = BP.LM(yr[p:p + win], yn[p:p + win], learning_rate, col_W, # J) # y_target,y_neuron, learning rate, col_W, Jacobian(col_X) """Sliding window""" for k in range(win + p, L): """Re-Learn every x samples, here we relearn every day""" if k % (re_learn) == 0 and k >= win + p: # Retrain # XX = X[k - win-p+1:k-p+1, :] # J = XNU_B.Jacobian(XX) J = JAC[k - win - p + 1:k - p + 1, :] e = np.zeros(win) for epoch in range(epoch2): for i in range(win): # klouzajici okno j = J[i, :] yn_temp = XNU_S.Y(j, col_W) e[i] = yr[k - win + 1 + i] - yn_temp col_W = BP.NGD(yr[k - win + 1 + i], yn_temp, learning_rate, j, col_W) SSE = np.append(SSE, np.sum((e**2))) #SSE[epoch] = sum((e) ** 2) print('Finished: ' + str(round((100.0 * ((k * 1.0 - win + 1)) / (L + 1 - win)), 2)) + '%') """For prediction we only need one row of samples as they include all reccurent values""" j = JAC[k, :] temporary = XNU_S.Y(j, col_W) yn[k + p] = temporary return yn, SSE
def CGD_batch(type, X, yr, p, win, re_learn, epoch1, epoch2): ''' :param type: 'CNU','QNU','LNU' :param X: vstupni matice :param yr: namerene hodnoty :param p: predikce :param win: uciciho sirka okna :param re_learn: po kolika vzorcich preucit :param epoch1: pocet uceni pro predtrenovani :param epoch2: pocet epoch uceni pro pretrenovavani :return: ''' start = time.time() # Zacatek mereni L = int(yr.shape[0] ) # sirka vektoru namerenych hodnot - musi byt stejna jako pro X X = nn.input_shape(X) # Uprava vektoru/matice viz neural_network.py yr = nn.output_shape(yr) # Uprava vektoru viz neural_network.py """The dimension of neuron and all matrixes like weights and so on depends on number of columns of input vector""" DIM = nn.input_dim( X[0, :] ) # X je vstupni matice, proto X[0, :] je vstupni vektor do neuronu. Touto funkci ziskam sirku vs. vektoru """Creating Neuron object with particular dimension""" if type == 'CNU': XNU_B = nn.HONU().CNU_Batch(DIM) # vznik objektu neuronu daneho typu col_W = XNU_B.col_W_from_W( XNU_B.W_CNU_init ) # ziskani dlouheho plocheho vektoru vah z matice vah if type == 'QNU': XNU_B = nn.HONU().QNU_Batch(DIM) col_W = XNU_B.col_W_from_W(XNU_B.W_QNU_init) if type == 'LNU': XNU_B = nn.HONU().LNU_Batch(DIM) col_W = XNU_B.col_W_from_W(XNU_B.W_LNU_init) """Creating Method object(BackPropagation) with particular method""" BP = nn.Backpropagation() # vytvoreni objektu ucicich algorimu BP CGD = BP.CGD() # Vytvoreni objektu z BP pro Conjugate Gradient Descent yn = np.zeros(L + p) # inicializace vektoru neuronoveho vystupu JAC = XNU_B.Jacobian( X[:, :] ) # Vytvoreni Jakobianu z cele vstupni matice - kvuli kratsimu vypoctu #J = XNU_B.Jacobian(X[:win,:]) J = JAC[:win, :] # J - kratkodoby jakobian - pro predtrenovani A = CGD.A(J) # Symetricka ctvercova matice pro CGD - poc. promenne b = CGD.b(yr[p:p + win], J) # poc. promena pro CGD re = CGD.re(b, A, col_W) # poc. promena pro CGD pp = re # poc. promena pro CGD Wm, EA, EAP, OLEs = nn.Learning_entropy(col_W, 4, L, p, 15) # alfas = np.array([10, 5, 3, 1, 0.5, 0.01]) * 10 # iterace - velmi rychly vypocet, neni treba pocitat vystup neuronu for i in range(epoch1): col_W, pp, re = CGD.CGD( A, col_W, re, pp) # y_target,y_neuron, learning rate, col_W, Jacobian(col_X) yn[p:p + win] = XNU_B.Y(J, col_W) """Sliding window- metoda predikce klouzaveho okna zacina zde""" k = 0 con = 1 for k in range( win + p, L ): # konec okna klouze od win+p po L, tedy posledni vypoctena hodnota je L+p """Re-Learn every x samples""" if k % ( re_learn ) == 0 and k >= win + p + 1: # Podminka, kdy pretrenovat - k%x = modulo, #XX = X[k-win-p+1:k-p+1,:] #J = XNU_B.Jacobian(XX) J = JAC[k - win - p + 1:k - p + 1, :] # kratkodoby jakobian A = CGD.A(J) # # poc. promena pro CGD b = CGD.b(yr[k - win + 1:k + 1], J) # poc. promena pro CGD re = CGD.re(b, A, col_W) # poc. promena pro CGD pp = re # poc. promena pro CGD print('Finished: ' + str(round((100.0 * ((k * 1.0 - win + 1)) / (L + 1 - win)), 2)) + '%') con = 1 # Iterace CGD for epoch in range(epoch2): col_W, pp, re = CGD.CGD(A, col_W, re, pp) """For prediction we only need one row of samples as they include all reccurent values""" #XX = X[k:k+1,:] # staci X[k-1,:], takto je temporary.shape = 1, # Jde nam o posledni hodnotu, tedy k+p, proto staci udelat kratkodoby jakobian posledniho radku J = JAC[k:k + 1, :] #J = XNU_B.Jacobian(XX) temporary = XNU_B.Y(J, col_W) # vystup je cislo yn[k + p] = temporary[ -1] # zbytecne v tomto pripade - duvod je ten ze je moznost brat treba poslednich 5 hodnot z vypocitaneho vektoru, tady se y pocita kazdy krok(k). Takze pokud krok bude 5, tak by bylo dobre brat 5 poslednich hodnot z vypocitaneho vektory y if con == 1: # con = 1 po pretrenovani # operace pro vypocet entropie - zpomaluje ale vypocet neuronove site Wm = nn.Wm_operation(Wm, col_W) con = 0 EA[k, :] = nn.fcnEA(Wm, alfas, OLEs) # LE EAP[k, :] = EAP[k - 1, :] + EA[k, :] else: EA[k, :] = 0 # LE EAP[k, :] = EAP[k - 1, :] + EA[k, :] return yn, EA, EAP