def predict():
configs = json.load(open(CONFIG, 'r'))
data = DataLoader(DATA, configs['data']['train_test_split'],
configs['data']['columns'])
global model
if model == None:
model = Model()
model.load_model(MODEL)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
if TYPE == "sequence":
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
if TYPE == "point" or TYPE == "predict":
predictions = model.predict_point_by_point(x_test)
if TYPE == "full":
predictions = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
if TYPE == "full" or TYPE == "point":
plot_results(predictions, y_test)
if TYPE == "predict":
predicted_value = data.denormalize_windows(
predictions[-1], configs['data']['sequence_length'])
sys.stdout.write("--END--{}--END--\n".format(predicted_value))
else:
sys.stdout.write("--END--")
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
if not os.path.exists(configs['model']['log_dir']):
os.makedirs(configs['model']['log_dir'])
data_loader = DataLoader(os.path.join('data',
configs['data']['filename_train']),
configs['data']['train_test_split'],
configs['data']['columns'],
is_training=True)
model = Model()
model.build_model(configs)
steps_per_epoch = math.ceil(
(data_loader.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
validation_steps = math.ceil(
(data_loader.len_val - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(train_loader=data_loader.batch_generator(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise'],
generator_type='train'),
val_loader=data_loader.batch_generator(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise'],
generator_type='val'),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps,
save_dir=configs['model']['save_dir'],
log_dir=configs['model']['log_dir'])
test_data_loader = DataLoader(os.path.join(
'data', configs['data']['filename_test']),
0,
configs['data']['columns'],
is_training=False)
x_test, y_test = test_data_loader.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
# predictions = model.predict_sequences_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
plot = Plot()
plot.plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
#獲取數據
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
#建立模型
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
#out-of memory generative training
#每一輪的'前傳導-后傳導'組合數量
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
#plot_results(predictions, y_test)
#if __name__=='__main__':
# main()
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
rates_count = configs['data']['sequence_length'] * configs['data'][
'number_sequences']
#os.path.join('data', configs['data']['filename']),
data = DataLoader(configs['data']['symbol'],
configs['data']['train_test_split'],
configs['data']['columns'], rates_count)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
#plot_results(predictions, y_test)
input("Press Enter to continue...")
def main(model_name=None):
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
if model_name:
saved_model = os.path.join(configs['model']['save_dir'], model_name)
model.load_model(saved_model)
else:
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
if not model_name:
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
def main():
configs = json.load(open('config.json', 'r'))
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
ndim=configs['model']['layers'][0]['input_dim'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
ndim=configs['model']['layers'][0]['input_dim'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
ndim=configs['model']['layers'][0]['input_dim'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main():
configs = json.load(open('config_battery_0630.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
configs['data']['sequence_length'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# # out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
#
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main():
configs = json.load(open(CONFIG, 'r'))
data = DataLoader(DATA, configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
model_path=MODEL)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
#plot_results(predictions, y_test)
sys.stdout.write("--END--")
def main(choice):
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
if (choice != 'info'):
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'])
# out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen = data.generate_train_batch(
# seq_len = configs['data']['sequence_length'],
# batch_size = configs['training']['batch_size'],
# normalise = configs['data']['normalise']
# ),
# epochs = configs['training']['epochs'],
# batch_size = configs['training']['batch_size'],
# steps_per_epoch = steps_per_epoch
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
if (choice == "multi"):
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
elif (choice == "seq"):
predictions = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
else:
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
def main():
#load parameters
configs = json.load(open('./data/config.json','r'))
if not os.path.exists(configs['model']['save_dir']):os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data',configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
)
#create RNN model
model=Model()
model.build_model(configs)
#loading trainning data
x,y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
print(x.shape)
print(y.shape)
#training model
model.train(
x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir']
)
#test results
x_test, y_test = data.get_test_data(
seq_len= configs['data']['sequence_length'],
normalise=configs['data']['normalise'],
)
#results visualization
predictions_multiseq = model.predict_sequences_multiple(x_test,configs['data']['sequence_length'],configs['data']['sequence_length'])
predictions_pointbypoint=model.predict_point_by_point(x_test)
plot_results_multiple(predictions_multiseq,y_test,configs['data']['sequence_length'])
plot_results(predictions_pointbypoint,y_test)
def main():
#读取所需参数
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
#读取数据
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
#创建RNN模型
model = Model()
mymodel = model.build_model(configs)
plot_model(mymodel, to_file='model.png', show_shapes=True)
#加载训练数据
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
print(x.shape)
print(y.shape)
#训练模型
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
#测试结果
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#展示测试效果
predictions = model.predict_sequences_multiple(
x_test,
configs['data']['sequence_length'],
configs['data']['sequence_length'],
debug=False)
print(np.array(predictions).shape)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main():
do = 4
if do == 0:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'],
X_test=x_test,
Y_test=y_test,
saveName=str(configs['training']['epochs']))
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
'''
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
# plot_results(predictions, y_test)
elif do == -1:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'],
X_test=x_test,
Y_test=y_test,
saveName=str(configs['training']['epochs']))
elif do == 1:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
model = Model()
model.build_model(configs)
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
model.load_model(
r"C:\Users\chris\Documents\Bitcoin_Prediction\saved_models\10092020-152418-e31.h5"
)
#configs['data']['sequence_length'] = 12
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
#predictions = model.predict_point_by_point(x_test)
plot_results_multiple(
predictions, y_test, configs['data']
['sequence_length']) #configs['data']['sequence_length']
#plot_results(predictions, y_test)
elif do == 2:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
model = Model()
model.build_model(configs)
model.load_model(
r"C:\Users\chris\Documents\Bitcoin_Prediction\saved_models\23092020-232350-e31.h5"
)
normed_test_data = pd.read_csv(
r"C:\Users\chris\Documents\Bitcoin_Prediction\data\2hdatarecent.csv"
)
normed_test_data = normed_test_data.get(configs['data']['columns'])
normed_test_data = normed_test_data[
-configs['data']['sequence_length'] + 1:]
norm_train_data = pd.read_csv(
r"C:\Users\chris\Documents\Bitcoin_Prediction\data\2hdatarecent.csv"
)
norm_train_data = norm_train_data.get(configs['data']['columns'])
norm_train_data = norm_train_data[-configs['data']['sequence_length'] +
1:]
normed_test_data = DataLoader.normalise_windows2(
model, window_data=normed_test_data, single_window=True)
norm_train_data = DataLoader.normalise_windows2(
model, window_data=norm_train_data, single_window=True)
print(normed_test_data)
test = data.currentData
model.predict_sequences_multipleSecondAttempt(test, 30)
else:
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
model = Model()
model.build_model(configs)
data = DataLoader(os.path.join('data', configs['data']["recentName"]),
configs['data']['train_test_split'],
configs['data']['columns'])
model.load_model(
r"C:\Users\chris\Documents\Bitcoin_Prediction\saved_models_Test\27122020-171426-e101testv3.h5"
)
testdata, printData = data.normData(seq_len=30, normalise=True)
predictions = model.predict_sequences_multipleSecondAttempt(
testdata, 29)
plot_results_multiple(predictions, printData, 29)
plt.show()
# %% Build/Train the model
model = Model()
model.build_model(configs)
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
def main():
# instantiation
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size']
)
'''
# out-of memory generative training
# 不懂为什么要自己算这个，LSTM不是自己自带batch参数吗，为什么要自己算出来一共要输多少次batch？？？
# 会出现：
# in data_generator_task， generator_output = next(self._generator)， StopIteration
# in fit_generator， str(generator_output))
# output of generator should be a tuple `(x, y, sample_weight)` or `(x, y)`. Found: None
# 所以出错的时候，手动减少steps_per_epoch
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size']) - 7
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
y_true_train, y_true_test = data.get_split_data()
unnorm_data = predict_unnorm_data(
y_true_test,
prediction_len=configs['data']['sequence_length'],
predictions=predictions)
# 计算RMSE并输出dataframe
begin_date = datetime(year=2018, month=9, day=18)
end_date = begin_date + timedelta(
days=(configs['data']['sequence_length'] - 1))
# y_true_test：（301，2）
y_true_test = pd.DataFrame(y_true_test)
file = pd.read_csv(os.path.join('data', configs['data']['filename']))
file = file['time'][len(y_true_train):]
file = pd.Series(file)
# 出现了无法新建列并赋值的error
# 因为dataframe和Series都有自己的index，.values才能取到真正的值并赋给下一个变量
y_true_test['time'] = file.values
y_true_test = y_true_test.set_index('time')
y_true_test.index = pd.to_datetime(y_true_test.index)
calc_RMSE(predicted_data=unnorm_data,
y_test_true=y_true_test,
begin_date=begin_date,
end_date=end_date)
def main():
config_path = 'config/config.json'
with open(config_path, 'r') as f:
configs = json.load(f)
logging.info("Loaded {}".format(config_path))
logging.info("\n{}\n".format(configs))
data_path = configs['data']['filename']
data_dir = os.path.dirname(data_path)
dtypes = configs['data'].get('dtypes', None)
windowed_normalization = configs['data']['normalise']
data = DataLoader(data_path,
configs['data']['train_test_split'],
configs['data']['columns'],
scaler_path=os.path.join(data_dir, "scaler"),
windowed_normalization=windowed_normalization,
dtypes=dtypes)
model = Model()
if configs['model'].get('load_model'):
model_path = os.path.join(configs['model']['load_model'])
logging.info("Loading {}".format(model_path))
model.load_model(model_path, configs)
plot_dir = os.path.join(os.path.dirname(model_path), "plots")
os.makedirs(plot_dir, exist_ok=True)
else:
plot_dir = os.path.join(configs['model']['save_dir'], "plots")
os.makedirs(plot_dir, exist_ok=True)
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
windowed_normalization=windowed_normalization)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of-memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
windowed_normalization=windowed_normalization),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
windowed_normalization=windowed_normalization)
predictions_multiple = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions_multiple,
y_test,
configs['data']['sequence_length'],
out_path=os.path.join(plot_dir, "multiple.png"))
def lstm(sTicker, postn, sFile, file_to_save):
configs = json.load(open('config.json', 'r'))
# print("hello")
# "How can I safely create a nested directory in Python?"
# stackoverflow.com/questions/273192/how-can-i-safely-create-a-nested-directory-in-python
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join(os.getcwd(), file_to_save),
configs['data']['train_test_split'],
configs['data']['columns'])
try:
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'],
file_name=sTicker)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
prediction_data = predictions[-1:]
prediction_data = prediction_data[0]
normalizer = data.data_test[-50, [0]]
# print(normalizer)
finaloutput = []
# print(prediction_data)
for i in range(len(prediction_data)):
finaloutput.append((prediction_data[i] + 1) * normalizer[0])
# print(finaloutput)
numpyFinal = np.array(finaloutput)
save_future(sTicker, numpyFinal)
plot_results_triple(sFile, finaloutput, file_to_save, sTicker,
predictions, y_test,
configs['data']['sequence_length'])
# plot_results_multiple(sTicker, predictions, y_test, configs['data']['sequence_length'])
# plot_results(sTicker, predictions, y_test)
totalDiff = [0] * len(predictions[0])
for x in range(0, 49):
y = y_test[(len(y_test) - 49) +
x] - predictions[len(predictions) - 1][x]
if y < 0.0:
y *= -1
totalDiff[x] = y
stdDev = float(np.std(totalDiff, 0))
nPred = predictions[len(predictions) - 1]
closeVal1 = pd.read_csv('stock_market_data.csv')
closeVal2 = closeVal1['Close']
closeVal3 = closeVal2[len(closeVal2) - 1]
avgPerc = 0
for deviate in nPred:
avgPerc += float(deviate)
divisor = len(nPred)
nMod1 = avgPerc / divisor
latPerc = nPred[len(nPred) - 1] - nPred[len(nPred) - 2]
nMod2 = nMod1 + 1
nMod3 = latPerc + 1
dPred1 = closeVal3 * nMod2
dPred2 = closeVal3 * nMod3
sFile['price_latest'][postn] = "{0:.2f}".format(closeVal3)
sFile['percent_change_avg'][postn] = "{0:.6f}".format(nMod1)
sFile['percent_change_latest'][postn] = "{0:.6f}".format(latPerc)
sFile['prediction_by_avg'][postn] = "{0:.2f}".format(dPred1)
sFile['prediction_by_latest'][postn] = "{0:.2f}".format(dPred2)
sFile['standard_deviation'][postn] = "{0:.6f}".format(stdDev)
except:
sFile['price_latest'][postn] = "n/a"
sFile['percent_change_avg'][postn] = "n/a"
sFile['percent_change_latest'][postn] = "n/a"
sFile['prediction_by_avg'][postn] = "n/a"
sFile['prediction_by_latest'][postn] = "n/a"
sFile['standard_deviation'][postn] = "n/a"
updatedFile = pd.DataFrame({
'stock_ticker':
sFile['stock_ticker'],
'percent_change_avg':
sFile['percent_change_avg'],
'percent_change_latest':
sFile['percent_change_latest'],
'prediction_by_avg':
sFile['prediction_by_avg'],
'prediction_by_latest':
sFile['prediction_by_latest'],
'price_latest':
sFile['price_latest'],
'standard_deviation':
sFile['standard_deviation']
})
updatedFile.to_csv("stock_performance.csv", index=False, header=True)
def main():
code_file=open("app/controllers/temp/code.txt","r")
#code_file=open("/root/stock/app/controllers/temp/code.txt","r")
the_code = code_file.read().splitlines()
code_file.close()
the_code=''.join(the_code)
the_code=the_code.zfill(6)
input_file = "app/controllers/temp/history_" + the_code + ".csv"
output_file = "app/controllers/temp/future/future_" + the_code +".csv"
#input_file = "/root/stock/app/controllers/temp/data/history/history_" + the_code + ".csv"
#output_file = "/root/stock/app/controllers/temp/future/future_" + the_code +",csv"
configs = json.load(open('app/controllers/temp/future/config.json', 'r'))
data = DataLoader(
input_file,
configs['data']['train_test_split'],
configs['data']['columns']
)
#model = Model()
model_close = Model()
model_open = Model()
model_high = Model()
model_low = Model()
#model.load_model("app/controllers/temp/future/model.h5")
model_close.load_model("app/controllers/temp/future/close.h5")
model_open.load_model("app/controllers/temp/future/open.h5")
model_high.load_model("app/controllers/temp/future/high.h5")
model_low.load_model("app/controllers/temp/future/low.h5")
x_test, y_test, data_raw = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
x_close = x_test[:,:,[0,4]]
x_open = x_test[:,:,[1,4]]
x_high = x_test[:,:,[2,4]]
x_low = x_test[:,:,[3,4]]
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions_close = model_close.predict_sequences_multiple(x_close, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions_open = model_open.predict_sequences_multiple(x_open, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions_high = model_high.predict_sequences_multiple(x_high, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions_low = model_low.predict_sequences_multiple(x_low, configs['data']['sequence_length'], configs['data']['sequence_length'])
predicted=[]
# for i, data in enumerate(predictions):
# predicted=predicted+data
predicted_close=[]
for i, data in enumerate(predictions_close):
predicted_close=predicted_close+data
predicted_open=[]
for i, data in enumerate(predictions_open):
predicted_open=predicted_open+data
predicted_high=[]
for i, data in enumerate(predictions_high):
predicted_high=predicted_high+data
predicted_low=[]
for i, data in enumerate(predictions_low):
predicted_low=predicted_low+data
predicted.append(predicted_close)
predicted.append(predicted_open)
predicted.append(predicted_high)
predicted.append(predicted_low)
data_raw = np.array(data_raw).astype(float)
denormalised_data = []
predicted = np.array(predicted).astype(float)
# for i in range(len(predicted)):
# denormalised_col = (float(predicted[i])+1)*float(data_raw[i, 0, 0])
# denormalised_data.append(denormalised_col)
for i in range(predicted.shape[0]):
denormalised_window = []
for j in range(predicted.shape[1]):
denormalised_col = (float(predicted[i,j])+1)*float(data_raw[j,i, 0])
denormalised_window.append(denormalised_col)
denormalised_data.append(denormalised_window)
denormalised_data = np.array(denormalised_data).T
#print denormalised_data
name = ['close','open','high','low']
test = pd.DataFrame(columns=name,data=denormalised_data)
test.to_csv(output_file)
os.remove(input_file)
