def main(train_after=False):
config_file = 'web_flask/LSTM/config.json'
configs = json.load(open(config_file, 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(configs['data']['filename'],
configs['data']['train_test_split'],
configs['data']['columns'],
normalise_meth=configs['data']['normalise'])
model = Model()
model.build_model(configs) if not train_after else \
model.load_model(os.path.join( configs['model']['save_dir'],configs['model']['model_name']))
history = LossHistory()
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'],
history=history,
x_test=x_test,
y_test=y_test)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
'''
history.loss_plot('epoch')
#loss, accuracy = model.model.evaluate(x_test, y_test)
#print(loss,accuracy)
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x[0]) #_test)
#plot_results_multiple(predictions, y, configs['data']['sequence_length'])
plot_results(predictions, y)
def main():
configs = json.load(open("config.json", "r"))
if not os.path.exists(configs["model"]["save_dir"]):
os.makedirs(configs["model"]["save_dir"])
data = DataLoader(
os.path.join("data", configs["data"]["filename"]),
configs["data"]["train_test_split"],
configs["data"]["columns"],
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs["data"]["sequence_length"],
normalise=configs["data"]["normalise"],
)
"""
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
"""
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs["data"]["sequence_length"])
/ configs["training"]["batch_size"]
)
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs["data"]["sequence_length"],
batch_size=configs["training"]["batch_size"],
normalise=configs["data"]["normalise"],
),
epochs=configs["training"]["epochs"],
batch_size=configs["training"]["batch_size"],
steps_per_epoch=steps_per_epoch,
save_dir=configs["model"]["save_dir"],
)
x_test, y_test = data.get_test_data(
seq_len=configs["data"]["sequence_length"],
normalise=configs["data"]["normalise"],
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs["data"]["sequence_length"])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
#獲取數據
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
#建立模型
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
#out-of memory generative training
#每一輪的'前傳導-后傳導'組合數量
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
#plot_results(predictions, y_test)
#if __name__=='__main__':
# main()
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
if not configs['training']['train']:
model.load_model(filepath='saved_models/02102019-164727-e2.h5')
else:
model.train(
x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir']
)
# out-of memory generative training
# steps_per_epoch = math.ceil(
# (data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'],
# configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
rates_count = configs['data']['sequence_length'] * configs['data'][
'number_sequences']
#os.path.join('data', configs['data']['filename']),
data = DataLoader(configs['data']['symbol'],
configs['data']['train_test_split'],
configs['data']['columns'], rates_count)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
#plot_results(predictions, y_test)
input("Press Enter to continue...")
def main():
configs = json.load(open('config.json', 'r'))
#create folder for save model params
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
#plot true data
#plot_results(data.data_train,True)
#train model
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
predictions = model.predict_point_by_point(x_test)
# plot_results(predictions, y_test)
# print (predictions)
# plot_results(predictions, y_test)
data1 = pd.DataFrame(predictions)
data1.to_csv('predict.csv')
data2 = pd.DataFrame(y_test)
data2.to_csv('true.csv')
def main(model_name=None):
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
if model_name:
saved_model = os.path.join(configs['model']['save_dir'], model_name)
model.load_model(saved_model)
else:
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
if not model_name:
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main_sin():
config = json.load(open("config_sin.json", 'r'))
data = DataLoader(os.path.join('data', config['data']['filename']),
config['data']['train_test_split'],
config['data']['columns'])
x_train, y_train = data.get_train_data(config['data']['sequence_length'],
config['data']['normalise'])
x_test, y_test = data.get_test_data()
model = Model()
model.build_model(config)
model.train(x_train, y_train, config['training']['epochs'],
config['training']['batch_size'])
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
# 从已经保存的模型中加载模型，此时不需要再进行模型训练：即不需要再执行model.train()部分
# model.load_model(r'saved_models/15102019-155115-e2.h5')
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# print('x的shape是：{0}'.format(x.shape)) # (3942, 49, 2)
# print('y的shape是：{0}'.format(y.shape)) # (3942, 1)
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
'''
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
def main():
configs = json.load(open('config.json', 'r'))
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size']
)
'''
# out-of memory generative training
# math.ceil(所有窗的个数 / batch_size)
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch
)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'],
configs=configs)
x_test, y_test, p0 = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
y_test = np.reshape(np.copy(y_test), -1)
plot_results((p0 * (predictions + 1))[-200:], (p0 * (y_test + 1))[-200:])
measure_performance(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
x_test, y_test, onedot = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(onedot)
with open('output.txt', 'w') as f:
f.write('预测下一时间的螺栓螺母消耗量为：' + str(int((predictions[-1] + 1) * data.last_raw_data(seq_len=configs['data']['sequence_length']))))
def main(choice):
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
if (choice != 'info'):
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'])
# out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen = data.generate_train_batch(
# seq_len = configs['data']['sequence_length'],
# batch_size = configs['training']['batch_size'],
# normalise = configs['data']['normalise']
# ),
# epochs = configs['training']['epochs'],
# batch_size = configs['training']['batch_size'],
# steps_per_epoch = steps_per_epoch
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
if (choice == "multi"):
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
elif (choice == "seq"):
predictions = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
else:
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
def main():
configs = json.load(open(CONFIG, 'r'))
data = DataLoader(DATA, configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
model_path=MODEL)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
#plot_results(predictions, y_test)
sys.stdout.write("--END--")
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):os.makedirs(configs['model']['save_dir'])
model = Model()
my_model = model.build_model(configs)
plot_model(my_model, to_file='output\model.png', show_shapes=True)
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
x, y = data.get_train_data(
configs['data']['sequence_length'],
configs['data']['normalise']
)
print(x.shape)
print(y.shape)
print(configs['training']['batch_size'])
print(configs['model']['save_dir'])
model.train(x,
y,
configs['training']['epochs'],
configs['training']['batch_size'],
configs['model']['save_dir']
)
x_test, y_test = data.get_test_data(
configs['data']['sequence_length'],
configs['data']['normalise']
)
# predictions = model.predict_sequences_multiplt(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequences_full(x_test, configs['data']['sequence_length'])
prediction_point = model.predict_point_by_point(x_test)
# print(prediction_point)
# print(np.array(predictions).shape)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(prediction_point, y_test)
def main():
#load parameters
configs = json.load(open('./data/config.json','r'))
if not os.path.exists(configs['model']['save_dir']):os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data',configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
)
#create RNN model
model=Model()
model.build_model(configs)
#loading trainning data
x,y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
print(x.shape)
print(y.shape)
#training model
model.train(
x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir']
)
#test results
x_test, y_test = data.get_test_data(
seq_len= configs['data']['sequence_length'],
normalise=configs['data']['normalise'],
)
#results visualization
predictions_multiseq = model.predict_sequences_multiple(x_test,configs['data']['sequence_length'],configs['data']['sequence_length'])
predictions_pointbypoint=model.predict_point_by_point(x_test)
plot_results_multiple(predictions_multiseq,y_test,configs['data']['sequence_length'])
plot_results(predictions_pointbypoint,y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
)
model = Model()
model.build_model(configs)
# get train data
x, y = data.get_train_data()
#x=x.squeeze()
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# # out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# batch_size=configs['training']['batch_size'],
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
# testing model
x_test, y_test = data.get_test_data()
#x_test=x_test.squeeze()
predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training (heavier computation)
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#predictions are made point by point with model.predict_point_by_point
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
def main():
#读取所需参数
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
#读取数据
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
#创建RNN模型
model = Model()
mymodel = model.build_model(configs)
plot_model(mymodel, to_file='model.png', show_shapes=True)
#加载训练数据
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
print(x.shape)
print(y.shape)
#训练模型
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
#测试结果
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#展示测试效果
predictions = model.predict_sequences_multiple(
x_test,
configs['data']['sequence_length'],
configs['data']['sequence_length'],
debug=False)
print(np.array(predictions).shape)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
dataframe = pd.read_csv(configs['data']['filename'])
f = open('/Users/yucheng/Downloads/project2/stockIDs.txt', 'r')
stockIDs = [int(line.split('\n')[0]) for line in f.readlines()]
for id in stockIDs[377:378]:
# for id in stockIDs[444:500]:
print("index: ", stockIDs.index(id))
data = DataLoader(dataframe, id, configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(id=id,
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
def main():
configs = json.load(open('configcrops.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# Yogyakarta: Kulon progo, bantul, gunung kidul, sleman, DIY
# Jawa Barat: Bandung, Tasikmalaya, Majalengka, Cirebon, Kuningan, Garut, Sumedang, Cianjut, Subang, Purwakarta, Indramayu
# Ciamis, Sukabumi, Bogor, Bekasi, Karawang
# # out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
# # save_dir = configs['model']['save_dir']
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# print(x_test)
# print(y_test)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions_point = model.predict_point_by_point(x_test)
print(len(predictions_point))
plot_results(predictions_point, y_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
# predictions_full = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# plot_results(predictions_full, y_test)
groundtrue = data._groundtruths(1)
groundtrue = (groundtrue.ravel())
print(len(groundtrue))
RMSElist = []
for i in range(len(groundtrue)):
errorrate = groundtrue[i] - predictions_point[i]
hasilkuadrat = errorrate * errorrate
RMSElist.append(hasilkuadrat)
RMSE = sum(RMSElist) / (len(predictions_point) - 2)
RMSE = RMSE**(1 / 2)
print(RMSE)
getdataforecast = data._forecasting(5, 1)
total_prediksi = 5
takefrom = 5
forecast_result = model.forecast(total_prediksi, getdataforecast, takefrom)
# print(forecast_result[0])
# forecast_result=np.append(forecast_result,[0.0])
# print(forecast_result)
n_steps = 8
# split into samples
X, y = split_sequence(forecast_result, n_steps)
# reshape from [samples, timesteps] into [samples, timesteps, features]
n_features = 1
# print(X)
X = X.reshape((X.shape[0], X.shape[1], n_features))
# define model
model = Sequential()
model.add(LSTM(50, activation='relu', input_shape=(n_steps, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit(X, y, epochs=200, verbose=0)
# demonstrate prediction
for j in range(total_prediksi):
getxlastnumber = array(forecast_result[(-n_steps - 1):-1])
x_input = getxlastnumber
# print(x_input)
x_input = x_input.reshape((1, n_steps, n_features))
yhat = model.predict(x_input, verbose=0)
# print(yhat[0][0])
forecast_result = np.append(forecast_result, yhat[0])
# prediction_point=np.append(prediction_point,yhat[0])
plot_results_onlypredicted(forecast_result)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
# -- Data preparation: --
data = DataLoader(os.path.join('../data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test, p0_vec = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# -- Init and fit CNN model: --
n_features = x.shape[2]
n_steps = configs['data']['sequence_length'] - 1
# Define model
model = Sequential()
model.add(
Conv1D(filters=128,
kernel_size=2,
activation='linear',
input_shape=(n_steps, n_features)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(50, activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(1))
# Compile model
model.compile(optimizer=configs['model']['optimizer'],
loss=configs['model']['loss'])
# Fit model
timer = Timer()
timer.start()
print('[Model] Training Started')
model.fit(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'])
timer.stop()
print('[Model] Predicting one step ahead...')
# Get predictions
yhat = model.predict(x_test, verbose=0)
# Denormalize & plot
p_pred, p_true = denorm_transform(p0_vec, yhat, y_test)
plot_results(p_pred, p_true) #de-normalised, i.e., original fex units
# Compute evaluation metrics
assess = EvalMetrics(p_true, p_pred)
MAE = assess.get_MAE()
RMSE = assess.get_RMSE()
print("MAE on validation set is: %f" % MAE)
print("RMSE on validation set is: %f" % RMSE)
# Save model
save_dir = configs['model']['save_dir']
save_fname = os.path.join(
save_dir, '%s_cnn.h5' % (dt.datetime.now().strftime('%d%m%Y-%H%M%S')))
model.save(save_fname)
print(f'current xp,yp {self.xp, self.yp}')
def get_optimal_params(self):
return self.xp[np.argmax(self.yp)]
def main():
pass
if __name__ == '__main__':
main()
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
bolstm = BayesOptimizerLSTM(configs, model, data, init_params=[[1], [.1]])
bolstm.fit()
plt.plot(bolstm.xp, bolstm.yp)
plt.show()
def main():
configs = json.load(open('config.json', 'r'))
#==================== selection =====================#
if configs['mode']['selection'] == True:
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
#IDs = configs['data']['IDs']
with open(
'D:\ColumbiaCourses\Advanced Big Data Analytics 6895\milestone3\LSTM-Neural-Network-for-Time-Series-Prediction\data\ID.csv',
newline='') as f:
reader = csv.reader(f)
IDs = list(reader)
IDs = [x[0] for x in IDs]
model = Model()
if configs['mode']['train_new_model'] == True:
model.build_model(configs)
print('[Model] Training Started')
cnt = 0
#===== train ====#
for ID in IDs:
cnt += 1
filename = str(ID) + '.csv'
data = DataLoader(filename=os.path.join('data', filename),
split=configs['data']['train_test_split'],
cols=configs['data']['columns'],
test_only=False)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
if cnt % 1 == 0:
tocheckpoint = True
else:
tocheckpoint = False
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator_all(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'],
tocheckpoint=tocheckpoint,
ID=ID)
print('[Model] Training All Finished')
else:
model.load_model(configs['mode']['train_file_path'])
#===== predict =====#
print('[Prediction]Start to predict and rank')
ranklist = []
for ID in IDs:
print('predicting %s'.format(ID))
filename = str(ID) + '.csv'
data = DataLoader(filename=os.path.join('data', filename),
split=configs['data']['train_test_split'],
cols=configs['data']['columns'],
test_only=False)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_point_by_point(x_test)
test_score = score(y_true=y_test, y_pred=predictions)
ranklist.append((ID, *test_score))
ranklist.sort(key=lambda x: x[1])
with open("ranklist.csv", "w", newline="") as f:
writer = csv.writer(f)
writer.writerows(ranklist)
return
#====================================================#
#==================== single task ===================#
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
configs['mode']['test_only'] #############################
)
model = Model()
if configs['mode']['test_only'] == True:
model.load_model(configs['mode']['test_file_path'])
else:
if configs['mode']['train_new_model'] == True:
model.build_model(configs)
else:
model.load_model(configs['mode']['train_file_path'])
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'],
mode=configs['mode'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['prediction_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
test_score = score(y_true=y_test, y_pred=predictions)
# plot_results_multiple(predictions, y_test, configs['data']['prediction_length'])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
lossesMINE = []
lossesKERAS = []
# for day_prediction in [1, 2, 3, 4, 5, 10, 50]:
day_prediction = 10
print("Predicting %i days..." % day_prediction)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'],
day_pred=day_prediction)
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise'],
# day_pred=day_prediction
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'],
day_pred=day_prediction)
# print(x_test.shape)
# print(len(data.denormalization_vals))
# print(y_test.shape)
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
#
# y_test_unormalized = np.zeros((y_test.shape[0], ))
# prediction_unormalized = []
#
# for i in range(4):
# for j in range(int(configs['data']['sequence_length']) - 10):
# y_test_unormalized[j*(i+1)] = (y_test[j] + 1)*data.data_test[i*int(configs['data']['sequence_length']), 0]
# prediction_unormalized.append((predictions[j*(i+1)] + 1)*data.data_test[i*int(configs['data']['sequence_length']), 0])
npPredictions = np.asarray(predictions)
# print(type(npPredictions))
# print(type(y_test))
# print(npPredictions.shape)
# print(y_test.shape)
loss = 0
for i in range(len(npPredictions)):
loss += (npPredictions[i] - y_test[i])**2
print(loss)
keras_loss = model.model.evaluate(x_test, y_test)
print(keras_loss)
lossesMINE.append(loss)
lossesKERAS.append(keras_loss)
#plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
real_y = np.reshape(y_test, (y_test.shape[0], )) * np.asarray(
data.denormalization_vals) + np.asarray(data.denormalization_vals)
real_pred = predictions * np.asarray(
data.denormalization_vals) + np.asarray(data.denormalization_vals)
# print(real_y.shape)
# print(real_pred.shape)
data.denormalization_vals = []
#plot_results(predictions, y_test)
plot_results(real_pred, real_y)
print(lossesMINE)
print(lossesKERAS)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
model = Model()
model.build_model(configs)
#get live sensor data from Arduino and predict next 10 sensor data
sensor_port = serial.Serial('COM7', 9600)
sensor_port.close()
sensor_port.open()
seq_len = configs['data']['sequence_length'],
sensor_data = []
predictions_data = []
live_data = np.arange(seq_len[0] - 1)
plt.ion() #real time graph
while True:
i = 0
while i < seq_len[0] - 1: # store incoming data to testing data array
b = sensor_port.readline() # read a byte string
live_data[i] = float(b.decode())
sensor_data.append(live_data[i])
i += 1
sensor_struct_data = live_data[
np.newaxis, :, np.newaxis] #contruct live data for LSTM
predictions = model.predict_sequence_live(
sensor_struct_data, configs['data']['sequence_length']
) #Shift the window by 1 new prediction each time, re-run predictions on new window
predictions_data.append(predictions)
plot_results(predictions_data[-120:], sensor_data[-100:])
plt.show()
plt.pause(0.1) #critical to display continous img
#predict every 10 seq_len
#if len(sensor_data) > 1 * seq_len[0]:
#train every 100 seq_len
if len(sensor_data) > 10 * seq_len[0]:
np.savetxt('data\sensor.csv',
sensor_data,
delimiter=',',
header='sensor_value')
#load data for training
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
sensor_data = sensor_data[-100:]
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
# model.build_model(configs)
model.load_model("saved_models/dow_30_50%.h5")
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
print("x_test.shape")
print(x_test.shape)
predictions = model.predict_point_by_point(x_test)
########################################################################
from sklearn.metrics import mean_squared_error
# loss_final = mean_squared_error(predictions, y_test)
# print("Testing Loss = " + str(loss_final))
########################################################################
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
print(predictions.shape)
print(y_test.shape)
m = pd.DataFrame(predictions)
n = pd.DataFrame(y_test)
m.to_csv("predictions.csv")
n.to_csv("y_test.csv")
p = 0
t = 0
t_1 = 0
count = 0
for a in range(len(predictions)):
if (a == 0):
t_1 = y_test[a]
continue
'''
1 1 1 1 1 1 1 1 1
1 1 1 1 1 1 1 1 1
'''
p = predictions[a]
t = y_test[a]
match = (t - t_1) * (p - t_1)
if (match > 0):
count += 1
t_1 = t
print("Good prediction rate = " + str(count / len(predictions)))
plot_results(predictions, y_test)
def main():
# instantiation
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size']
)
'''
# out-of memory generative training
# 不懂为什么要自己算这个，LSTM不是自己自带batch参数吗，为什么要自己算出来一共要输多少次batch？？？
# 会出现：
# in data_generator_task， generator_output = next(self._generator)， StopIteration
# in fit_generator， str(generator_output))
# output of generator should be a tuple `(x, y, sample_weight)` or `(x, y)`. Found: None
# 所以出错的时候，手动减少steps_per_epoch
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size']) - 7
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
y_true_train, y_true_test = data.get_split_data()
unnorm_data = predict_unnorm_data(
y_true_test,
prediction_len=configs['data']['sequence_length'],
predictions=predictions)
# 计算RMSE并输出dataframe
begin_date = datetime(year=2018, month=9, day=18)
end_date = begin_date + timedelta(
days=(configs['data']['sequence_length'] - 1))
# y_true_test：（301，2）
y_true_test = pd.DataFrame(y_true_test)
file = pd.read_csv(os.path.join('data', configs['data']['filename']))
file = file['time'][len(y_true_train):]
file = pd.Series(file)
# 出现了无法新建列并赋值的error
# 因为dataframe和Series都有自己的index，.values才能取到真正的值并赋给下一个变量
y_true_test['time'] = file.values
y_true_test = y_true_test.set_index('time')
y_true_test.index = pd.to_datetime(y_true_test.index)
calc_RMSE(predicted_data=unnorm_data,
y_test_true=y_true_test,
begin_date=begin_date,
end_date=end_date)
def main():
config_path = 'config/config.json'
with open(config_path, 'r') as f:
configs = json.load(f)
logging.info("Loaded {}".format(config_path))
logging.info("\n{}\n".format(configs))
data_path = configs['data']['filename']
data_dir = os.path.dirname(data_path)
dtypes = configs['data'].get('dtypes', None)
windowed_normalization = configs['data']['normalise']
data = DataLoader(data_path,
configs['data']['train_test_split'],
configs['data']['columns'],
scaler_path=os.path.join(data_dir, "scaler"),
windowed_normalization=windowed_normalization,
dtypes=dtypes)
model = Model()
if configs['model'].get('load_model'):
model_path = os.path.join(configs['model']['load_model'])
logging.info("Loading {}".format(model_path))
model.load_model(model_path, configs)
plot_dir = os.path.join(os.path.dirname(model_path), "plots")
os.makedirs(plot_dir, exist_ok=True)
else:
plot_dir = os.path.join(configs['model']['save_dir'], "plots")
os.makedirs(plot_dir, exist_ok=True)
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
windowed_normalization=windowed_normalization)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of-memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
windowed_normalization=windowed_normalization),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
windowed_normalization=windowed_normalization)
predictions_multiple = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions_multiple,
y_test,
configs['data']['sequence_length'],
out_path=os.path.join(plot_dir, "multiple.png"))
def main():
do = 4
if do == 0:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'],
X_test=x_test,
Y_test=y_test,
saveName=str(configs['training']['epochs']))
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
'''
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
# plot_results(predictions, y_test)
elif do == -1:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'],
X_test=x_test,
Y_test=y_test,
saveName=str(configs['training']['epochs']))
elif do == 1:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
model = Model()
model.build_model(configs)
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
model.load_model(
r"C:\Users\chris\Documents\Bitcoin_Prediction\saved_models\10092020-152418-e31.h5"
)
#configs['data']['sequence_length'] = 12
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(
predictions, y_test, configs['data']
['sequence_length']) #configs['data']['sequence_length']
#plot_results(predictions, y_test)
elif do == 2:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
model = Model()
model.build_model(configs)
model.load_model(
r"C:\Users\chris\Documents\Bitcoin_Prediction\saved_models\23092020-232350-e31.h5"
)
normed_test_data = pd.read_csv(
r"C:\Users\chris\Documents\Bitcoin_Prediction\data\2hdatarecent.csv"
)
normed_test_data = normed_test_data.get(configs['data']['columns'])
normed_test_data = normed_test_data[
-configs['data']['sequence_length'] + 1:]
norm_train_data = pd.read_csv(
r"C:\Users\chris\Documents\Bitcoin_Prediction\data\2hdatarecent.csv"
)
norm_train_data = norm_train_data.get(configs['data']['columns'])
norm_train_data = norm_train_data[-configs['data']['sequence_length'] +
1:]
normed_test_data = DataLoader.normalise_windows2(
model, window_data=normed_test_data, single_window=True)
norm_train_data = DataLoader.normalise_windows2(
model, window_data=norm_train_data, single_window=True)
print(normed_test_data)
test = data.currentData
model.predict_sequences_multipleSecondAttempt(test, 30)
else:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
model = Model()
model.build_model(configs)
data = DataLoader(os.path.join('data', configs['data']["recentName"]),
configs['data']['train_test_split'],
configs['data']['columns'])
model.load_model(
r"C:\Users\chris\Documents\Bitcoin_Prediction\saved_models_Test\27122020-171426-e101testv3.h5"
)
testdata, printData = data.normData(seq_len=30, normalise=True)
predictions = model.predict_sequences_multipleSecondAttempt(
testdata, 29)
plot_results_multiple(predictions, printData, 29)
