def start_train():
config = {"batch": 256, "epochs": 10}
file_path1=fileStr1.get()
file_path2='data/100211data/100211_all_test.csv'
if file_path1=="" or file_path2=="":
tkinter.messagebox.askokcancel(title='请选择文件~', message='请选择两个文件')
return
print("start_train")
callBack = keras.callbacks.LambdaCallback(
on_epoch_end=lambda epoch, logs: print("epoch",epoch)
)
needLstm =modelName.get()=="lstm" or modelName.get()=="all"
needAllDense = modelName.get()=="allDense" or modelName.get()=="all"
# _thread.start_new_thread(show_progress,())
if needLstm:
for i in range(lagIntStart.get(),lagIntEnd.get(),2):
lag = i
X_train, y_train, _, _, _ = process_data(file_path1, file_path2, lag)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_lstm([lag, 64, 64, 1])
train_model(m, X_train, y_train, "lstm", config,lag,callBack)
if needAllDense:
for i in range(lagIntStart.get(), lagIntEnd.get(), 2):
lag=i
X_train, y_train, _, _, _ = process_data(file_path1, file_path2, lag)
m = model.get_AllDense([lag, 64, 64, 1])
train_allDense_model(m, X_train, y_train, "AllDense", config, lag,callBack)
tkinter.messagebox.askokcancel(title='ok~', message='训练完成，结果保存在model文件夹下')
return
def checkTestError():
lstm = load_model('model/lstm.h5')
gru = load_model('model/gru.h5')
saes = load_model('model/saes.h5')
models = [lstm, gru, saes]
names = ['LSTM', 'GRU', 'SAEs']
lag = 12
train = 'data/train/'
test = 'data/test/'
_, _, X_test, y_test = process_data(train, test)
y_preds = []
for name, model in zip(names, models):
print(name)
if name == 'SAEs':
X_test = np.reshape(X_test, (X_test.shape[0], -1))
file = 'images/' + name + '.png'
#plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
j = 1
for i in range(len(predicted)):
print('文件', j, ': ')
j += 1
eva_regress(y_test[i], np.array(predicted[i]))
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument(
"--model",
default="saes",
help="Model to train.")
args = parser.parse_args()
lag = 12
config = {"batch": 256, "epochs": 50}
file1 = '/home/VICOMTECH/icejudo/PROYECTOS/MiRepoGithub/MLflow-dvc-data/data/train.csv'
file2 = '/home/VICOMTECH/icejudo/PROYECTOS/MiRepoGithub/MLflow-dvc-data/data/test.csv'
X_train, y_train, _, _, _ = process_data(file1, file2, lag)
if args.model == 'lstm':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_lstm([12, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'gru':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_gru([12, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'saes':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1]))
m = model.get_saes([12, 400, 400, 400, 1])
train_seas(m, X_train, y_train, args.model, config)
def main():
lstm = load_model('model/lstm.h5')
#gru = load_model('model/gru.h5')
#saes = load_model('model/saes.h5')
models = [lstm] #, gru, saes
names = ['LSTM'] #, 'GRU', 'SAEs'
lag = 12
file1 = 'data/train.csv'
file2 = 'data/test.csv'
_, _, X_test, y_test, scaler, Min, Max = process_data(file1, file2, lag)
y_test = y_test * (Max - Min) + Min
#scaler.inverse_transform(y_test.reshape(-1, 3)).reshape(1, -1)[0]
y_preds = []
for name, model in zip(names, models):
if name == 'SAEs':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
else:
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
#plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test) * (Max - Min) + Min
#predicted = scaler.inverse_transform(predicted.reshape(-1, 3)).reshape(1, -1)[0]
y_preds.append(predicted[:288])
print(name)
eva_regress(y_test, predicted)
plot_results(y_test[:288], y_preds, names)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument(
"--model",
default="lstm",
help="Model to train.")
args = parser.parse_args()
lag = 12
config = {"batch": 256, "epochs": 600}
file1 = 'data/train.csv'
file2 = 'data/test.csv'
X_train, y_train, _, _, _ = process_data(file1, file2, lag)
if args.model == 'lstm':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_lstm([12, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'gru':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_gru([12, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'saes':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1]))
m = model.get_saes([12, 400, 400, 400, 1])
train_seas(m, X_train, y_train, args.model, config)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument("--model", default="ann", help="Model to train.")
args = parser.parse_args()
lag = 6
config = {"batch": 256, "epochs": 200}
file1 = 'data/compareData/train_data_compare(after6am).csv'
file2 = 'data/compareData/test_data_cluster0_compare(after6am).csv'
X_train, y_train, X_test, y_test, scaler = process_data(file1, file2, lag)
if args.model == 'lstm':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_lstm([6, 100, 100, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'ann':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1]))
m = model.get_ann([6, 20, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'gru':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_gru([6, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'saes':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1]))
m = model.get_saes([6, 100, 100, 100, 1])
train_seas(m, X_train, y_train, args.model, config)
def main(type):
file1 = 'data/train.csv'
file2 = 'data/test.csv'
"""
num_of_input =2 : weekofyear, weekday
num_of_input = 3: weekofyear, weekday, hour
num_of_input = 4: month, day, hour, minute
"""
X_train, y_train, _, _, _ = process_data(file1, file2, type)
step = 50
# epochs_range =range(8,20)
# number_input_range = range(2, 10)
# epochs_range =range(10, 14)
# number_input_range = range(5, 7)
# epochs_range =range(18, 21)
# batchs_range =range(4, 8)
# number_input_range = range(6, 10)
# for i in epochs_range:
# for j in number_input_range:
# for b in batchs_range:
# t = multiprocessing.Process(target=run_thread, args=(X_train, y_train, int(math.pow(2,b)), step*i, int(math.pow(2,j)), ))
# t.start()
# t.join()
run_thread(X_train, y_train, 32, 1000, 256)
def main():
lstm = load_model('model/lstm.h5')
nn = load_model('model/nn.h5')
gru = load_model('model/gru.h5')
saes = load_model('model/saes.h5')
models = [nn, lstm, saes, gru]
names = ['nn', 'lstm', 'saes', 'gru']
file1 = 'data/train.csv'
file2 = 'data/test.csv'
_, _, X_test, y_test, scaler = process_data(file1, file2)
y_preds = []
for name, model in zip(names, models):
if name == 'nn':
pass
elif name == 'saes':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
print('11')
else:
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
y_preds.append(predicted)
print(name)
eva_regress(y_test, predicted)
plot_results(y_test, y_preds, names)
def main():
gru = load_model('model/gru.h5')
lstm = load_model('model/lstm.h5')
saes = load_model('model/saes.h5')
models = [lstm, gru, saes]
names = ['LSTM', 'GRU', 'SAEs']
lag = 12
file1 = 'data/train.csv'
file2 = 'data/test.csv'
_, _, X_test, y_test, scaler = process_data(file1, file2, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
for name, model in zip(names, models):
if name == 'SAEs':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
else:
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[:288])
print(name)
eva_regress(y_test, predicted)
# print(y_test[: 288],y_test)
# 这个288决定了要绘制哪一天的图像
plot_results(y_test[:288], y_preds, names)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument(
"--model",
default="bidirectional",
help="Model to train.")
args = parser.parse_args()
lag = 4 # how far to look back
config = {"batch": 256, "epochs": 15 }
file1 = './data/train1.csv'
file2 = './data/test1.csv'
X_train, y_train, _, _, _ = process_data(file1, file2, lag)
if args.model == 'lstm':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_lstm([4, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'bidirectional':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_bidirectional([4, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'simplernn':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_simplernn([4, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'gru':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
m = model.get_gru([4, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config)
if args.model == 'saes':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1]))
m = model.get_saes([4, 400, 400, 400, 1])
train_seas(m, X_train, y_train, args.model, config)
def main():
lstm = load_model('model/lstm4_layers_4.h5')
gru = load_model('model/gru4_layers_4.h5')
saes = load_model('model/saes4_layers_4.h5')
simple_rnn = load_model('model/simplernn4_layers_4.h5')
models = [lstm, gru, saes, simple_rnn]
names = ['LSTM', 'GRU', 'SAEs', 'SIMPLE_RNN']
lag = 4
file1 = './data/train1.csv'
file2 = './data/test1.csv'
_, _, X_test, y_test, scaler = process_data(file1, file2, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
for name, model in zip(names, models):
if name == 'SAEs':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
else:
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(1, -1)[0]
y_preds.append(predicted[:96])
print(name)
eva_regress(y_test, predicted)
plot_results(y_test[: 96], y_preds, names)
def main():
lstm = load_model('model/lstm.h5')
gru = load_model('model/gru.h5')
saes = load_model('model/saes.h5')
models = [lstm, gru, saes]
names = ['LSTM', 'GRU', 'SAEs']
lag = 12
file1 = 'data/train.csv'
file2 = 'data/test.csv'
_, _, X_test, y_test, scaler = process_data(file1, file2, lag)
# 将已经规格化的在进行一次反规格化，得到原始数据
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
os.environ["PATH"] += os.pathsep + r'D:\document\graphviz\bin'
for name, model in zip(names, models):
if name == 'SAEs':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
else:
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[:288])
print(name)
# y_test 真实数据，全部 predicated 预测，全部
eva_regress(y_test, predicted)
plot_results(y_test[:288], y_preds, names)
def __init__(self, raw_data, location, units, lags, config):
self.raw_data = raw_data
self.location = location
self.units = units
self.lags = lags
self.config = config
self.X_train, self.y_train, self.X_test, self.y_test, self.scaler = process_data(
self.raw_data, self.location, self.lags)
def train_with_args(scats, junction, model_to_train):
""" Start the training process with specific arguments
Parameters:
scats (int): the scats site identifier
junction (int): the VicRoads internal id for the location
model_to_train (String): the neural network model to train
"""
config = get_setting(
"train"
) # Get the config, e.g: {'lag': 12, 'batch': 256, 'epochs': 600}
print(f"(train.py) CONFIG: {config}")
file_directory = 'model/' + model_to_train
if scats != "All":
junctions = SCATS_DATA.get_scats_approaches(
scats) # Get array of scats approaches, e.g: [1, 3, 5, 7]
print(f"(train.py) SCATS SITES: {junctions}")
file_directory = f"{file_directory}/{scats}/"
filename = junction
if junction != "All": # If the junction in args is not all...
junctions = [junction]
print(f"(train.py) SCATS SITES: {junctions}"
) # ... set args to be the junctions e.g.: ['1']
# TODO: Determine if strings are an issue here
for junction in junctions:
print("Training {0}/{1} using a {2} model...".format(
scats, junction, model_to_train))
x_train, y_train, _, _, _ = process_data(scats, junction,
config["lag"])
else:
file_directory = f"{file_directory}/Generalised/"
filename = "Model"
print("Training a generalised {0} model...".format(model_to_train))
x_train, y_train = SCATS_DATA.get_training_data()
scats_site = "All"
junction = "All"
print(
f"(train.py) XTRAIN[0]: {x_train[0][:10]} \n XTRAIN[1]: {x_train[1][:10]} \n YTRAIN: {y_train[:10]}"
)
print(
f"(traint.py) XTRAIN SHAPE: {x_train.shape} \n YTRAIN SHAPE: {y_train.shape}"
)
if os.path.isfile(f"{file_directory}{filename}.h5"):
m = load_model(f"{file_directory}{filename}.h5")
else:
input_shape = (x_train.shape[1], )
m = generate_new_model(model_to_train, input_shape)
if model_to_train == 'seas':
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1]))
train_seas(m, x_train, y_train, file_directory, filename, config)
else:
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
train_model(m, x_train, y_train, file_directory, filename, config)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument("--scats", default=970, help="SCATS site number.")
parser.add_argument("--junction",
default=1,
help="The approach to the site.")
args = parser.parse_args()
models = []
untrained_models = []
model_names = ['LSTM', 'GRU', 'SAEs', 'FEEDFWD', 'DEEPFEEDFWD']
""" Getting the trained models is split into two parts
because of some issues when removing items from a list that is being iterated over """
for name in model_names:
# Construct the path to the file
file = "model/{0}/{1}/{2}.h5".format(name.lower(), args.scats,
args.junction)
if os.path.exists(file):
models.append(load_model(file))
else:
untrained_models.append(name)
for name in untrained_models:
# Remove all untrained models so they are not included on the graph
model_names.remove(name)
lag = get_setting("train")["lag"]
_, _, x_test, y_test, scaler = process_data(args.scats, args.junction, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
mtx = []
for name, model in zip(model_names, models):
if name == 'SAEs':
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1]))
else:
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(x_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[:96])
print(f"X_TEST: {x_test[0]}")
print(name)
mtx.append(eva_regress(y_test, predicted))
plot_results(y_test[:96], y_preds, model_names)
plot_error(mtx)
mae = "mae"
print(f"\nMTX: {mtx}")
print(f"\n{mtx[0][mae]}")
print(f"\n{mtx[0].keys()}")
def printPlot(model):
lstm = load_model('model/lstm.h5')
gru = load_model('model/gru.h5')
#saes = load_model('model/saes.h5')
train = 'data/train/'
test = 'data/test/'
_, _, X_test, y_test = process_data(train, test)
if model == 'saes':
X_test = np.reshape(X_test, (X_test.shape[0], -1))
#predicted = saes.predict(X_test)
if model == 'lstm':
predicted = lstm.predict(X_test)
if model == 'gru':
predicted = gru.predict(X_test)
arousal_true, valence_true, arousal_pred, valence_pred = [], [], [], []
for i in range(len(y_test)):
arousal_true.append(y_test[i][0])
valence_true.append(y_test[i][1])
arousal_pred.append(round(predicted[i][0], 2))
valence_pred.append(round(predicted[i][1], 2))
plt.figure()
plt.xlabel('Valence')
plt.ylabel('Arousal') # 设置坐标轴的文字标签
ax = plt.gca() # get current axis 获得坐标轴对象
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none') # 将右边 上边的两条边颜色设置为空 其实就相当于抹掉这两条边
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left') # 指定下边的边作为 x 轴 指定左边的边为 y 轴
ax.spines['bottom'].set_position(
('data', 0)) # 指定 data 设置的bottom(也就是指定的x轴)绑定到y轴的0这个点上
ax.spines['left'].set_position(('data', 0))
plt.title('LALALA', fontsize=22, color=(0.4, 0.4, 0.4), loc='center')
plt.plot(np.array(valence_true),
np.array(arousal_true),
linestyle='--',
label='origin')
plt.plot(np.array(valence_pred), np.array(arousal_pred), label='pred')
plt.xlim(xmin=-6, xmax=6)
plt.ylim(ymin=-6, ymax=6)
plt.legend(bbox_to_anchor=(1.05, 0), loc=0, borderaxespad=0)
plt.savefig('images/result.png')
def main(target_model):
config = {"batch": 10, "epochs": 25000}
train = 'data/train/'
test = 'data/test/'
X_train, y_train, _, _ = process_data(train, test)
if target_model == 'lstm':
m = model.get_lstm([58, 64, 64, 2])
train_model(m, X_train, y_train, target_model, config)
if target_model == 'gru':
m = model.get_gru([58, 64, 64, 2])
train_model(m, X_train, y_train, target_model, config)
if target_model == 'saes':
X_train = np.reshape(X_train, (X_train.shape[0], -1))
m = model.get_saes([4800, 400, 400, 400, 2])
train_seas(m, X_train, y_train, target_model, config)
def main():
lstm = load_model('model/lstm.h5')
gru = load_model('model/lstm.h5')
saes = load_model('model/saes.h5')
models = [lstm, gru, saes]
names = ['LSTM', 'GRU', 'SAEs']
lag = 12
file1 = 'data/train.csv'
file2 = 'data/test.csv'
X_train, y_train, X_test, y_test, scaler = process_data(file1, file2, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
for name, model in zip(names, models):
if name == 'SAEs':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
else:
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
# plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[:288])
print(name)
# print(predicted)
# print(type(predicted))
# print(predicted.shape)
# print(y_test)
eva_regress(y_test, predicted)
# Prophet
# df1=pd.DataFrame({'ds': X_train, 'y': y_train.tolist()}, columns=['ds', 'y'])
# df2=pd.DataFrame({'ds': X_test, 'y': y_test.tolist()}, columns=['ds', 'y'])
prophet_prediction = get_prophet(file1, file2)
print("Prophet")
eva_regress(y_test, prophet_prediction)
y_preds.append(prophet_prediction[:288])
names.append("Prophet")
plot_results(y_test[:288], y_preds, names)
def main(model_con, dataset, batch, epoch):
if not os.path.exists(PATH1 + os.sep + dataset.split(".")[0]):
os.makedirs(PATH1 + os.sep + dataset.split(".")[0])
parser = argparse.ArgumentParser()
parser.add_argument("--model", default=model_con, help="Model to train.")
args = parser.parse_args()
lag = 288
config = {"batch": batch, "epochs": epoch}
X_train, y_train, _, _, _ = process_data(FILE1, FILE2,
lag) #input, process data
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
if args.model == 'tcclstmlsm':
m = load_model('tcclstmlsm.h5')
#m = model.get_tcnlstm([288, 64, 64, 1])
train_model(m, X_train, y_train, args.model, config, dataset)
def main(dataset, lag):
tcclstmlsm = load_model('tcclstmlsm.h5')
models = [tcclstmlsm]
names = ['TCC-LSTM-LSM']
_, _, X_test, y_test, scaler = process_data(FILE1, FILE2, lag)
y_test = scaler.inverse_transform(y_test)
y_preds = []
for name, model in zip(names, models):
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted)
predicted = predicted[:, -1]
y_preds.append(predicted[:288])
print(name)
eva_regress(dataset, name, y_test[:, -1], predicted)
def main(argv):
config = {"batch": 256, "epochs": 600}
file1 = 'data/100211data/100211_all_train.csv'
file2 = 'data/100211data/100211_all_test.csv'
#得到不同lag的lstm model
# for i in range(16,18,2):
# lag = i
# X_train, y_train, _, _, _ = process_data(file1, file2, lag)
# X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
# m = model.get_lstm([lag, 64, 64, 1])
# train_model(m, X_train, y_train, "lstm", config,lag)
#得到全连接神经网络训练model(lag=12
lag = 12
X_train, y_train, _, _, _ = process_data(file1, file2, lag)
m = model.get_AllDense([lag, 64, 64, 1])
train_allDense_model(m, X_train, y_train, "AllDense", config, lag)
def main(argv):
parser=argparse.ArgumentParser()
parser.add_argument(
'--model',
default='lstm',
help='Model to train')
args=parser.parse_args()
lag=12
config={"batch": 256,"epochs":500}
path=r'D:\data\2018_data\process_data\M30f.csv'
if args.model in ['lstm','gru','saes']:
X_train,y_train,_,_,_=process_data(path,lag)
if args.model == 'merge_lstm':
lag = [12, 96, 672]
X_train, y_train, _, _, _ =min_day_week_data(path,lag)
# X_train = np.reshape(X_train,[X_train.shape[0], X_train.shape[1], 1])
m = model.merge_lstm([12,24,7,1])
train_merge_lstm(m, X_train, y_train, args.model, config)
if args.model=='lstm':
X_train=np.reshape(X_train,[X_train.shape[0],X_train.shape[1],1])
m=model.get_lstm([12,64,128,1])
train_model(m,X_train,y_train,args.model,config)
if args.model=='gru':
X_train=np.reshape(X_train,[X_train.shape[0],X_train.shape[1],1])
m=model.get_gru([12,64,128,1])
train_model(m,X_train,y_train,args.model,config)
if args.model=='saes':
X_train=np.reshape(X_train,[X_train.shape[0],X_train.shape[1]])
m=model.get_saes([12,400,400,400,1])
train_seas(m,X_train,y_train,args.model,config)
if args.model=='cnn':
lag=96
X_train,y_train,_,_,_=cnn_data(path,lag)
X_train=np.reshape(X_train,[X_train.shape[0],X_train.shape[1],X_train.shape[2],1])
# y_train=np.reshape(y_train,[y_train.shape[0],y_train.shape[1],1])
m=model.get_cnn([68,96,16,32,64])
train_model(m,X_train,y_train,args.model,config)
def main():
lstm = load_model('model/lstm.h5')
gru = load_model('model/gru.h5')
saes = load_model('model/saes.h5')
models = [gru]
names = ['GRU']
lag = 12
file1 = 'data/train.csv'
file2 = 'data/test.csv'
X_train, y_train, X_test, y_test, scaler = process_data(file1, file2, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
for name, model in zip(names, models):
if name == 'SAEs':
x_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
else:
x_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(x_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[:288])
print(name)
eva_regress(y_test, predicted)
####### for SVM #######
model = svm.SVR()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
y_pred = scaler.inverse_transform(y_pred.reshape(-1, 1)).reshape(1, -1)[0]
print('SVM')
eva_regress(y_test, y_pred)
names.append('SVM')
y_preds.append(y_pred[:288])
#######################
plot_results(y_test[:288], y_preds, names)
def main():
lstm = load_model('model/lstm.h5')
gru = load_model('model/gru.h5')
saes = load_model('model/saes.h5')
ann = load_model('model/ann.h5')
svr = svm.SVR(kernel='rbf')
# models = [lstm, gru, saes, ann, svr]
# names = ['LSTM', 'GRU', 'SAEs', 'ANN', 'SVR']
models = [svr]
names = ['SVR']
lag = 6
file1 = 'data/compareData/train_data_compare(after6am).csv'
file2 = 'data/compareData/test_data_cluster2_compare(after6am).csv'
X_train, y_train, X_test, y_test, scaler = process_data(file1, file2, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
for name, model in zip(names, models):
if name == 'SAEs' or name == 'ANN':
# SAEs的输入形状[n-lags,lags]
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
elif name == 'SVR':
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1]))
model.fit(X_train, y_train)
else:
# GRU、LSTM输入形状[n-lags,lags,1]
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
# 模型结构
if name != 'SVR':
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[:211])
print(name)
eva_regress(y_test, predicted)
plot_results(y_test[:211], y_preds, names)
def main():
lstm = load_model('model/lstm.h5')
gru = load_model('model/gru.h5')
saes = load_model('model/saes.h5')
merge_lstm = load_model('model/merge_lstm.h5')
models = [lstm, gru, merge_lstm]
names = ['LSTM', 'GRU', 'merge_lstm']
lag = 12
path = r'D:\data\2018_data\process_data\M30f.csv'
_, _, X_test, y_test, scaler = process_data(path, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_test = y_test[672:]
y_pred = []
os.environ['path'] += os.pathsep + r'D:\document\graphviz\bin'
for name, model in zip(names, models):
if name == 'merge_lstm':
lag = [12, 96, 672]
_, _, X_test, y_test, scaler = min_day_week_data(path, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(
1, -1)[0]
predicted = model.predict(X_test)
else:
X_test = np.reshape(X_test, [X_test.shape[0], X_test.shape[1], 1])
predicted = model.predict(X_test[672:])
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_pred.append(predicted[:96])
print(name)
eva_regress(y_test, predicted)
plot_results(y_test[:96], y_pred, names)
from data.data import process_data import numpy as np file1 = 'data/train/' file2 = 'data/test/' X_train, y_train, _, _ = process_data(file1, file2) X_train = np.reshape(X_train, (X_train.shape[0], -1)) print(X_train.shape)
def predict(self, my_scats, st, et, my_day, d, name):
lag = 4
# Filter data files on input values
file1 = './data/train1.csv'
file2 = './data/test1.csv'
train = pd.read_csv(file1)
test = pd.read_csv(file2)
error = 0
train = train[(train['SCATS'] == my_scats) & (train['DAY'] == my_day)
& (train['TIME'] >= st) & (train['TIME'] <= et) &
(train[d] > 0)]
test = test[(test['SCATS'] == my_scats) & (test['DAY'] == my_day)
& (test['TIME'] >= st) & (test['TIME'] <= et) &
(test[d] > 0)]
# If test data <= lag, return error
if (len(test.index) <= 4):
return error
else:
test.to_csv('./data/my_test.csv', encoding='utf-8', index=False)
train.to_csv('./data/my_train.csv', encoding='utf-8', index=False)
my_test = './data/my_test.csv'
my_train = './data/my_train.csv'
_, _, X_test, y_test, scaler = process_data(my_train, my_test, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds = []
if name == 'GRU':
model = self.models[0]
if name == 'LSTM':
model = self.models[1]
if name == 'SAES':
model = self.models[2]
if name == 'RNN':
model = self.models[3]
if name == 'BI':
model = self.models[4]
if name == 'SAES':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1]))
else:
X_test = np.reshape(X_test,
(X_test.shape[0], X_test.shape[1], 1))
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(
-1, 1)).reshape(1, -1)[0]
y_preds.append(predicted[:96])
self.metrics = self.eva_regress(y_test, predicted)
# Get average traffic count per hour
time_range = int((et - st) / 100)
count = 0
# Check predictions, if only 1 then return it
if (len(predicted) == 1):
count = predicted[0]
else:
for i in range(time_range):
count += predicted[i]
count_phour = np.int64(round((count / time_range), 0))
return count_phour
draw.iloc[7000:7500, 0].plot(figsize=(12, 6))
draw.iloc[7000:7500, 1].plot(figsize=(12, 6))
plt.legend(('real', 'predict'), fontsize='15')
plt.title("valid", fontsize='30') # 添加标题
plt.show()
if __name__ == '__main__':
gru = load_model('model/gru.h5', custom_objects={'identity_function': identity_function})
time_step, seq_len = 90, 9
path_dict = {"train": "data/train.csv", "valid": "data/valid.csv", "test": "data/test.csv"}
# read data
X_train, y_train, y_train_sec, X_valid, y_valid, y_valid_sec, X_test, y_test, y_test_sec, y_mean, y_std = \
process_data(path_dict, time_step, seq_len)
# run NN
y_train_predict = gru.predict(X_train)
y_valid_predict = gru.predict(X_valid)
y_test_predict = gru.predict(X_test)
y_train_predict = y_train_predict[:, -1] * y_std + y_mean + y_train_sec # 预测出的当前时刻90s后预测值
y_train = y_train * y_std + y_mean + y_train_sec # 当前时刻90s后的label值
y_valid_predict = y_valid_predict[:, -1] * y_std + y_mean + y_valid_sec
y_valid = y_valid * y_std + y_mean + y_valid_sec
y_test_predict = y_test_predict[:, -1] * y_std + y_mean + y_test_sec
y_test = y_test * y_std + y_mean + y_test_sec
def compareMLPAndLSTM():
lag = 10
lstm = load_model("model/lstm-" + str(lag) + ".h5")
allDense = load_model('model/AllDense-10.h5')
models = [allDense, lstm]
names = ['AllDense', 'lstm']
file1 = 'data/100211data/100211_weekend_train.csv'
file2 = 'data/100211data/100211_weekend_test.csv'
if file_path == "":
tkinter.messagebox.askokcancel(title='请选择文件~', message='请先选择测试集文件')
return
_, _, X_test, y_test, scaler = process_data(file1, file_path, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1, 1)).reshape(1, -1)[0]
y_preds = []
columnData = []
for name, model in zip(names, models):
if name == 'lstm':
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/' + name + '.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[0:288])
print(name)
evaValue = eva_regress(y_test, predicted)
columnData.append(evaValue)
plot_results(y_test[0:288], y_preds, names)
print(columnData)
print(np.array(columnData)[:, 0])
dataTable = tkinter.Toplevel()
dataTable.title("MLP和LSTM的训练参数比较")
dataTable.geometry("1600x1600")
# 创建表格
tree_date = ttk.Treeview(dataTable)
# 定义列
tree_date['columns'] = ['MLP', 'LSTM']
tree_date.pack()
# 设置列宽度
tree_date.column('MLP', width=200)
tree_date.column('LSTM', width=200)
tree_date.heading('MLP', text='MLP')
tree_date.heading('LSTM', text='LSTM')
# 给表格中添加数据
tree_date.insert('',
0,
text='EVS',
values=tuple(np.array(columnData)[:, 0]))
tree_date.insert('',
1,
text='MAPE',
values=tuple(np.array(columnData)[:, 1]))
tree_date.insert('',
2,
text='MAE',
values=tuple(np.array(columnData)[:, 2]))
tree_date.insert('',
3,
text='MSE',
values=tuple(np.array(columnData)[:, 3]))
img = Image.open('./images/pre_weekend_time.png')
img_png = ImageTk.PhotoImage(img)
label_img = ttk.Label(dataTable, image=img_png)
label_img.pack()
dataTable.mainloop()
def compareLSTMWithLag():
file1 = 'data/100211data/100211_weekend_train.csv'
file2 = 'data/100211data/100211_weekend_test.csv'
begin = xls_text2.get()
end = xls_text3.get()
if begin == "" or end == "":
tkinter.messagebox.askokcancel(title='请输入lag范围~', message='请先输入lag范围')
return
if file_path == "":
tkinter.messagebox.askokcancel(title='请选择文件~', message='请先选择测试集文件')
return
dataTable = tkinter.Toplevel()
dataTable.title("LSTM选取不同lag的训练参数比较")
dataTable.geometry("1600x1600")
# 创建表格
tree_date = ttk.Treeview(dataTable)
y_preds = []
columnData = []
names = []
begin = int(begin)
end = int(end)
columns = []
for lag in range(begin, end, 2):
columns.append('lag=' + str(lag))
model = load_model("model/lstm-" + str(lag) + ".h5")
_, _, X_test, y_test, scaler = process_data(file1, file_path, lag)
y_test = scaler.inverse_transform(y_test.reshape(-1,
1)).reshape(1, -1)[0]
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
file = 'images/lstm.png'
plot_model(model, to_file=file, show_shapes=True)
predicted = model.predict(X_test)
predicted = scaler.inverse_transform(predicted.reshape(-1, 1)).reshape(
1, -1)[0]
y_preds.append(predicted[0:288])
print("lag=", lag)
names.append("lstm lag=" + str(lag))
evaValue = eva_regress(y_test, predicted)
columnData.append(evaValue)
plot_results(y_test[0:288], y_preds, names)
print(columnData)
print(np.array(columnData)[:, 0])
#根据输入的lag范围动态生成表格
tree_date['columns'] = columns
for lag in range(begin, end, 2):
tree_date.column('lag=' + str(lag), width=200)
tree_date.heading('lag=' + str(lag), text='lag=' + str(lag))
# 定义列
# tree_date['columns'] = ['lag=4','lag=6','lag=8','lag=10','lag=12','lag=14']
tree_date.pack()
# 给表格中添加数据
tree_date.insert('',
0,
text='EVS',
values=tuple(np.array(columnData)[:, 0]))
tree_date.insert('',
1,
text='MAPE',
values=tuple(np.array(columnData)[:, 1]))
tree_date.insert('',
2,
text='MAE',
values=tuple(np.array(columnData)[:, 2]))
tree_date.insert('',
3,
text='MSE',
values=tuple(np.array(columnData)[:, 3]))
img = Image.open('./images/pre_weekend_time.png')
img_png = ImageTk.PhotoImage(img)
label_img = ttk.Label(dataTable, image=img_png)
label_img.pack()
dataTable.mainloop()
