def main(config):
import time
T0 = time.time()
F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, Bx, By, Bz, E, Kp, Dst, ap, AE = dataprocess1(
)
data_in = np.hstack((F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, Bx, By,
Bz, E, Kp, Dst, ap, AE))
print('Modified data_in shape:', data_in.shape)
data_out = dataprocess2()
print('data_out shape:', data_out.shape)
data_in = data_in
Setpre = data_in
Labelpre = data_out
# multiprocess training
pool = multiprocessing.Pool(processes=7)
## seven years data, split into 7 parts. 2408 = 344*7
oneyear = 344
res = []
res2 = []
for i in range(7):
print('The %d year' % i)
yrange = np.arange(oneyear * i, oneyear * i + oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
Set = np.delete(Setpre, yrange, axis=0)
Label = np.delete(Labelpre, yrange, axis=0)
res.append(
pool.apply_async(supermodel1_2,
(Set, Label, ShowSet, ShowLabel, config)))
pool.close()
pool.join()
print('finished !!!')
print('res!!!', res)
for j in res:
x = j.get()
res2.append(x)
#--------------------------------------
seed = config.seed
res2 = pd.DataFrame(res2)
print('res2 shape:', res2)
path = './result2/'
res2.to_csv(path + 'FFN_prediction' + str(seed) + '.csv',
index=False,
header=None,
sep=',')
TT = time.time()
print('finally finished !!!', TT - T0)
def main2(config):
F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, Bx, By, Bz, E, Kp, Dst, ap, AE = dataprocess1(
)
data_in = np.hstack((F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, Bx, By,
Bz, E, Kp, Dst, ap, AE))
print('Modified data_in shape:', data_in.shape)
data_out = dataprocess2()
print('data_out shape:', data_out.shape)
data_in = data_in #/10
Setpre = data_in
Labelpre = data_out
## seven years data, split into 7 parts. 2408 = 344*7
oneyear = 344
DF1, Dy, Dy2 = [], [], []
for i in range(7):
print('The %d year' % i)
yrange = np.arange(oneyear * i, oneyear * i + oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
Set = np.delete(Setpre, yrange, axis=0)
Label = np.delete(Labelpre, yrange, axis=0)
df1, dy, dy2 = firstorder(Set, Label, ShowSet, ShowLabel, config)
DF1.append(df1)
Dy.append(dy)
Dy2.append(dy2)
DF1 = np.array(DF1).reshape(-1, 100)
Dy = np.array(Dy).reshape(-1)
Dy2 = np.array(Dy2).reshape(-1)
print('D shape:', DF1.shape, Dy.shape, Dy2.shape)
fx1 = np.sum(np.abs(np.array(DF1)), axis=0)
fx1 = NORM2(fx1)
Dx = list(np.sum(np.array(DF1), axis=1).reshape(-1))
print('Dx:', Dx)
rate1 = ratecal(Dx, Dy)
rate1_2 = ratecal(Dx, Dy2)
print('rate1', rate1, rate1_2)
# fisrt order information
fx1 = np.array(fx1).reshape(-1)
return fx1
import pandas as pd
import matplotlib.pyplot as plt
from preprocess import dataprocess2
def correct_rate(y_true, y_pred):
y_true = np.array(y_true).reshape(-1)
y_pred = np.array(y_pred).reshape(-1)
right_num = 0
for i in range(len(y_true)):
if y_true[i] <= y_pred[i]:
right_num += 1
return float(right_num)/len(y_true)
Label = dataprocess2()
Label += 8
print(Label.shape)
## single value prediction of quantile
RIdata = pd.read_csv('./result2/quan_log.csv', index_col=None, header=0, skiprows=268, nrows=133, sep=',')
RIdata = np.array(RIdata)[:, -16:].reshape(-1, 16*19)
print(RIdata.shape)
RIdata = np.mean(RIdata, axis=0).reshape(-1, 16)
pre = []
for i in range(11, 20):
file = 'Quan_'+str(i)+'.csv'
# 9*3*2408
predata = pd.read_csv('./result2/'+file, index_col=False, header=None, sep=',')
pre2 = np.array(pre2).reshape(-1)
pre3 = np.array(pre3).reshape(-1)
prediction = np.vstack((pre1, pre2, pre3))
TT = time.time()
print('finished!', TT - T0)
return prediction
if __name__ == '__main__':
F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, E, Kp, Dst, ap, AE = dataprocess1(
)
data_in = np.hstack(
(F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, E, Kp, Dst, ap, AE))
print('Modified data_in shape:', data_in.shape)
data_out = dataprocess2()
print('data_out shape:', data_out.shape)
Setpre = data_in
Labelpre = data_out
for j in range(10):
pre = []
config = config2()
config.seed += 1
for i in range(1, 10, 1):
if hasattr(config, 'taurange'):
config.taurange = [0.5 - 0.05 * i, 0.5, 0.5 + 0.05 * i]
prediction = alpha2cal(Setpre, Labelpre, config)
pre.append(prediction)
def main(config):
import time
T0 = time.time()
F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, E, Kp, Dst, ap, AE = dataprocess1(
)
data_in = np.hstack(
(F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, E, Kp, Dst, ap, AE))
print('Modified data_in shape:', data_in.shape)
data_out = dataprocess2()
print('data_out shape:', data_out.shape)
data_in = data_in
Setpre = data_in
Labelpre = data_out
# multiprocess training
pool = multiprocessing.Pool(processes=7)
## seven years data, split into 7 parts. 2408 = 344*7
oneyear = 344
res = []
res2 = []
weightall = []
for i in range(7):
print('The %d year' % i)
yrange = np.arange(oneyear * i, oneyear * i + oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
Set = np.delete(Setpre, yrange, axis=0)
Label = np.delete(Labelpre, yrange, axis=0)
#trloss, teloss, shloss, trainPE, testPE, showPE, trainCC, testCC, showCC, TT, \
# weight = supermodel1(Set, Label, ShowSet, ShowLabel, config)
res.append(
pool.apply_async(supermodel1,
(Set, Label, ShowSet, ShowLabel, config)))
#res.append([trloss, teloss, shloss, trainPE, testPE, showPE, TT])
#weightall.append(weight)
pool.close()
pool.join()
print('finished !!!')
print('res!!!', res)
for j in res:
x = j.get()
res2.append(x[:-1])
weightall.append(x[-1])
#--------------------------------------
run_num = config.run_num
seed = config.seed
learnrate = config.learnrate
epochs = config.epochs
batchsize = config.batch_size
reg1 = config.reg1
reg2 = config.reg2
num_in = config.num_in
node1 = config.node1
node2 = config.node2
weight_init = config.weight_init
f1 = config.fun1
f2 = config.fun2
#Ls = config.Loss
path = config.path0
logfile = config.logfile
res2 = pd.DataFrame(res2)
res2.to_csv(path + 'FFNres' + str(run_num) + '.csv',
index=False,
header=None,
sep=',')
weightall = pd.DataFrame(weightall)
res3 = pd.read_csv(path + 'FFNres' + str(run_num) + '.csv',
index_col=False,
header=None,
skiprows=0,
sep=',')
res_mean = res3.mean(axis=0)
res_mean = list(res_mean)
res_mean = [np.round(x, 6) for x in res_mean]
print('res_mean:', res_mean)
weightall_mean = weightall.mean(axis=0)
weightall_mean = list(weightall_mean)
weightall_mean = [np.round(x, 12) for x in weightall_mean]
print('weightall_mean:', weightall_mean)
# save data to logfile
FFN_para = pd.read_csv('./result2/' + logfile,
index_col=False,
header=0,
sep=',')
config_para = [
run_num, seed, epochs, batchsize, learnrate, reg1, reg2, num_in, node1,
node2, weight_init, f1, f2
]
finalres = config_para + res_mean + weightall_mean
# columns name
col1 = [
'run_num', 'seed', 'epochs', 'batchsize', 'learnrate', 'reg1', 'reg2',
'num_in', 'node1', 'node2', 'weight_init', 'fun1', 'fun2'
]
col2 = [
'trloss', 'teloss', 'shloss', 'trainPE', 'testPE', 'showPE', 'trainCC',
'testCC', 'showCC', 'TT'
]
col3 = []
for i in range(num_in):
col3 += ['w' + str(i + 1)]
finalcol = col1 + col2 + col3
# save data
lognum = len(FFN_para)
FFN_para.loc[lognum] = finalres
FFN_para.to_csv('./result2/' + logfile,
index=False,
header=finalcol,
sep=',')
TT = time.time()
print('finally finished !!!', TT - T0)
def main(config):
F1,F2,F3,F4,F5,F6,F7,T,N,V,P,Bt,E,Kp,Dst,ap,AE = dataprocess1()
data_in = np.hstack((F1,F2,F3,F4,F5,F6,F7,T,N,V,P,Bt,E,Kp,Dst,ap,AE))
print('Modified data_in shape:', data_in.shape)
data_out = dataprocess2()
print('data_out shape:', data_out.shape)
Setpre = data_in
Labelpre = data_out
# multiprocess training
pool = multiprocessing.Pool(processes=7)
## seven years data, split into 7 parts. 2408 = 344*7
oneyear = 344
res = []
trloss, teloss, shloss = [], [], []
pre1, pre2, pre3, pre4, pre5, pre6, pre7, pre8, pre9 = [], [], [], [], [], [], [], [], []
for i in range(7):
print('The %d year'%i)
yrange = np.arange(oneyear*i, oneyear*i+oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
Set = np.delete(Setpre, yrange, axis=0)
Label = np.delete(Labelpre, yrange, axis=0)
res.append(pool.apply_async(supermodel3_2, (Set, Label, ShowSet, ShowLabel, config)))
pool.close()
pool.join()
print('finished !!!')
print('res!!!', res)
for j in res:
x = j.get()
pre1.append(x[0])
pre2.append(x[1])
pre3.append(x[2])
pre4.append(x[3])
pre5.append(x[4])
pre6.append(x[5])
pre7.append(x[6])
pre8.append(x[7])
pre9.append(x[8])
trloss.append(x[9])
teloss.append(x[10])
shloss.append(x[11])
## calculate the correct rate
rate1, rate2, rate3, rate4, rate5, rate6, rate7, rate8, rate9 = [], [], [], [], [], [], [], [], []
for i in range(7):
print('The %d year'%i)
yrange = np.arange(oneyear*i, oneyear*i+oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
prediction1 = pre1[i]
prediction2 = pre2[i]
prediction3 = pre3[i]
prediction4 = pre4[i]
prediction5 = pre5[i]
prediction6 = pre6[i]
prediction7 = pre7[i]
prediction8 = pre8[i]
prediction9 = pre9[i]
rate1.append(correct_rate(ShowLabel, prediction1))
rate2.append(correct_rate(ShowLabel, prediction2))
rate3.append(correct_rate(ShowLabel, prediction3))
rate4.append(correct_rate(ShowLabel, prediction4))
rate5.append(correct_rate(ShowLabel, prediction5))
rate6.append(correct_rate(ShowLabel, prediction6))
rate7.append(correct_rate(ShowLabel, prediction7))
rate8.append(correct_rate(ShowLabel, prediction8))
rate9.append(correct_rate(ShowLabel, prediction9))
###---------------
run_num = config.run_num
seed = config.seed
learnrate = config.learnrate
epochs = config.epochs
batchsize = config.batch_size
reg1 = config.reg1
reg2 = config.reg2
num_in = config.num_in
node1 = config.node1
node2 = config.node2
weight_init = config.weight_init
f1 = config.fun1
f2 = config.fun2
logfile = './result2/quan_log3.csv'
para_col = ['run_num', 'seed', 'num_in', 'epochs', 'batchsize', 'learnrate', 'reg1', 'reg2', 'node1', 'node2',
'weight_init', 'fun1', 'fun2', 'trloss', 'teloss', 'shloss']
ratecol, ratedata = [], []
for i in range(1, 8, 1):
for j in range(1, 10, 1):
ratecol.append(str(i)+'_'+str(j))
ratedata = rate1+rate2+rate3+rate4+rate5+rate6+rate7+rate8+rate9
finalcol = para_col+ratecol
# save data to logfile
Quan_para = pd.read_csv(logfile, index_col=False, header=0, sep=',')
config_para = [run_num, seed, num_in, epochs, batchsize, learnrate, reg1, reg2, node1, node2, weight_init, f1, f2]
finalres = config_para+[np.mean(trloss), np.mean(teloss), np.mean(shloss)]+ratedata
lognum = len(Quan_para)
Quan_para.loc[lognum] = finalres
Quan_para.to_csv(logfile, index=False, header=finalcol, sep=',')
TT = time.time()
print('finished!!!', TT-T0)
def NORM2(data):
data = np.array(data).reshape(-1)
datasum = np.sum(data)
data = data / datasum
return data
if __name__ == '__main__':
F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, Bx, By, Bz, E, Kp, Dst, ap, AE = dataprocess1(
)
data_in = np.hstack((F1, F2, F3, F4, F5, F6, F7, T, N, V, P, Bt, Bx, By,
Bz, E, Kp, Dst, ap, AE))
print('Modified data_in shape:', data_in.shape)
data_out, _ = dataprocess2()
print('data_out shape:', data_out.shape)
Setpre = data_in
Labelpre = data_out
config = config()
#pool = multiprocessing.Pool(processes=7)
######-------------------------------------七年模型分年训练
oneyear = 343
yrange = np.arange(oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
Set = np.delete(Setpre, yrange, axis=0)
Label = np.delete(Labelpre, yrange, axis=0)
trainset = Set
def main(config):
F1,F2,F3,F4,F5,F6,F7,T,N,V,P,Bt,E,Kp,Dst,ap,AE = dataprocess1()
data_in = np.hstack((F1,F2,F3,F4,F5,F6,F7,T,N,V,P,Bt,E,Kp,Dst,ap,AE))
print('Modified data_in shape:', data_in.shape)
data_out = dataprocess2()
print('data_out shape:', data_out.shape)
Setpre = data_in
Labelpre = data_out
# multiprocess training
pool = multiprocessing.Pool(processes=7)
## seven years data, split into 7 parts. 2408 = 344*7
oneyear = 344
res = []
trloss, teloss, shloss = [], [], []
pre2, pre3 = [], []
for i in range(7):
print('The %d year'%i)
yrange = np.arange(oneyear*i, oneyear*i+oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
Set = np.delete(Setpre, yrange, axis=0)
Label = np.delete(Labelpre, yrange, axis=0)
res.append(pool.apply_async(supermodel3, (Set, Label, ShowSet, ShowLabel, config)))
pool.close()
pool.join()
print('finished !!!')
print('res!!!', res)
for j in res:
x = j.get()
trloss.append(x[0])
teloss.append(x[1])
shloss.append(x[2])
pre2.append(x[-2])
pre3.append(x[-1])
## calculate the correct rate
rate1, rate2 = [], []
for i in range(7):
print('The %d year'%i)
yrange = np.arange(oneyear*i, oneyear*i+oneyear)
ShowSet = Setpre[yrange, :]
ShowLabel = Labelpre[yrange, :]
Set = np.delete(Setpre, yrange, axis=0)
Label = np.delete(Labelpre, yrange, axis=0)
prediction2 = pre2[i]
prediction3 = pre3[i]
## attention, due to "//10", the datasize may not be consistent
prediction2_num = len(prediction2)
rate1.append(correct_rate(Label[0: prediction2_num], prediction2))
rate2.append(correct_rate(ShowLabel, prediction3))
###---------------
tau = config.tau
run_num = config.run_num
seed = config.seed
learnrate = config.learnrate
epochs = config.epochs
batchsize = config.batch_size
reg1 = config.reg1
reg2 = config.reg2
num_in = config.num_in
node1 = config.node1
node2 = config.node2
weight_init = config.weight_init
f1 = config.fun1
f2 = config.fun2
#Ls = config.Loss
logfile = './result2/quan_log.csv'
rate1_mean = np.mean(rate1)
rate2_mean = np.mean(rate2)
para_col = ['run_num', 'tau', 'seed', 'num_in', 'epochs', 'batchsize', 'learnrate', 'reg1', 'reg2', 'node1', 'node2',
'weight_init', 'fun1', 'fun2', 'trloss', 'teloss', 'shloss', 'testrate', 'showrate']
ratecol, ratedata = [], []
for i in range(1, 8, 1):
ratecol.append('t'+str(i))
ratecol.append('s'+str(i))
ratedata.append(rate1[i-1])
ratedata.append(rate2[i-1])
finalcol = para_col+ratecol
# save data to logfile
Quan_para = pd.read_csv(logfile, index_col=False, header=0, sep=',')
config_para = [run_num, tau, seed, num_in, epochs, batchsize, learnrate, reg1, reg2, node1, node2, weight_init, f1, f2]
finalres = config_para+[np.mean(trloss), np.mean(teloss), np.mean(shloss), rate1_mean, rate2_mean]+ratedata
lognum = len(Quan_para)
Quan_para.loc[lognum] = finalres
Quan_para.to_csv(logfile, index=False, header=finalcol, sep=',')
TT = time.time()
print('finished!!!', TT-T0)
