def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('../data', configs['data']['filename']),
os.path.join('../data', configs['data']['VIMfile']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# Out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test, p0_vec = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
pred = predictions.reshape((predictions.size, 1))
#plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
#plot_results(pred, y_test) #normalised predictions
# De-normalise & plot
p_pred, p_true = denorm_transform(p0_vec, pred, y_test)
plot_results(p_pred, p_true) #de-normalised, i.e., original fex units
# Compute evaluation metrics
assess = EvalMetrics(p_true, p_pred)
MAE = assess.get_MAE()
RMSE = assess.get_RMSE()
print("MAE on validation set is: %f" % MAE)
print("RMSE on validation set is: %f" % RMSE)
def predict_prepare(data_x = None,model_path = None,config_file = 'web_flask/LSTM/config.json'):
config_file = config_file
configs = json.load(open(config_file, 'r'))
'''
data_loader = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
normalise_meth=configs['data']['normalise']
)
# 用所有数据进行预测
data_x,data_y = data_loader.get_all_data(configs['data']['sequence_length'],\
normalise=configs['data']['normalise'])
data_x = data_x[-1]
'''
model = Model()
model_way = './web_flask/LSTM/saved_models/20052019-174244-e60.h5'
model.load_model(model_way)
predictions = model.predict_point_by_point(data_x)
# print(predictions)
''' 每五分钟预测一个值，貌似是错的
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataProcessor(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=".")
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions_pointbypoint = model.predict_point_by_point(x_test)
plot_results(predictions_pointbypoint, y_test)
predictions_fullseq = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
plot_results(predictions_fullseq, y_test)
def predict():
configs = json.load(open(CONFIG, 'r'))
data = DataLoader(DATA, configs['data']['train_test_split'],
configs['data']['columns'])
global model
if model == None:
model = Model()
model.load_model(MODEL)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
if TYPE == "sequence":
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
if TYPE == "point" or TYPE == "predict":
predictions = model.predict_point_by_point(x_test)
if TYPE == "full":
predictions = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
if TYPE == "full" or TYPE == "point":
plot_results(predictions, y_test)
if TYPE == "predict":
predicted_value = data.denormalize_windows(
predictions[-1], configs['data']['sequence_length'])
sys.stdout.write("--END--{}--END--\n".format(predicted_value))
else:
sys.stdout.write("--END--")
def main(train_after=False):
config_file = 'web_flask/LSTM/config.json'
configs = json.load(open(config_file, 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(configs['data']['filename'],
configs['data']['train_test_split'],
configs['data']['columns'],
normalise_meth=configs['data']['normalise'])
model = Model()
model.build_model(configs) if not train_after else \
model.load_model(os.path.join( configs['model']['save_dir'],configs['model']['model_name']))
history = LossHistory()
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'],
history=history,
x_test=x_test,
y_test=y_test)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
'''
history.loss_plot('epoch')
#loss, accuracy = model.model.evaluate(x_test, y_test)
#print(loss,accuracy)
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x[0]) #_test)
#plot_results_multiple(predictions, y, configs['data']['sequence_length'])
plot_results(predictions, y)
def main():
configs = json.load(open("config.json", "r"))
if not os.path.exists(configs["model"]["save_dir"]):
os.makedirs(configs["model"]["save_dir"])
data = DataLoader(
os.path.join("data", configs["data"]["filename"]),
configs["data"]["train_test_split"],
configs["data"]["columns"],
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs["data"]["sequence_length"],
normalise=configs["data"]["normalise"],
)
"""
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
"""
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs["data"]["sequence_length"])
/ configs["training"]["batch_size"]
)
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs["data"]["sequence_length"],
batch_size=configs["training"]["batch_size"],
normalise=configs["data"]["normalise"],
),
epochs=configs["training"]["epochs"],
batch_size=configs["training"]["batch_size"],
steps_per_epoch=steps_per_epoch,
save_dir=configs["model"]["save_dir"],
)
x_test, y_test = data.get_test_data(
seq_len=configs["data"]["sequence_length"],
normalise=configs["data"]["normalise"],
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs["data"]["sequence_length"])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
if not configs['training']['train']:
model.load_model(filepath='saved_models/02102019-164727-e2.h5')
else:
model.train(
x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir']
)
# out-of memory generative training
# steps_per_epoch = math.ceil(
# (data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'],
# configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
#create folder for save model params
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
#plot true data
#plot_results(data.data_train,True)
#train model
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
predictions = model.predict_point_by_point(x_test)
# plot_results(predictions, y_test)
# print (predictions)
# plot_results(predictions, y_test)
data1 = pd.DataFrame(predictions)
data1.to_csv('predict.csv')
data2 = pd.DataFrame(y_test)
data2.to_csv('true.csv')
def predict(test):
# initialize dataLoader with split of 0
cleaner.main_func()
data = DataLoader(test, 0, configs['data']['columns'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'], normalise=False)
model = Model()
model.load_model('saved_models/tracker.h5')
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
return "OK"
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
# 从已经保存的模型中加载模型，此时不需要再进行模型训练：即不需要再执行model.train()部分
# model.load_model(r'saved_models/15102019-155115-e2.h5')
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# print('x的shape是：{0}'.format(x.shape)) # (3942, 49, 2)
# print('y的shape是：{0}'.format(y.shape)) # (3942, 1)
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
'''
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#
# '''
# # in-memory training
# model.train(
# x,
# y,
# epochs = configs['training']['epochs'],
# batch_size = configs['training']['batch_size'],
# save_dir = configs['model']['save_dir']
# )
# '''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
predictions = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'],
configs=configs)
x_test, y_test, p0 = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
y_test = np.reshape(np.copy(y_test), -1)
plot_results((p0 * (predictions + 1))[-200:], (p0 * (y_test + 1))[-200:])
measure_performance(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
x_test, y_test, onedot = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(onedot)
with open('output.txt', 'w') as f:
f.write('预测下一时间的螺栓螺母消耗量为：' + str(int((predictions[-1] + 1) * data.last_raw_data(seq_len=configs['data']['sequence_length']))))
def main(choice):
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
if (choice != 'info'):
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'])
# out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen = data.generate_train_batch(
# seq_len = configs['data']['sequence_length'],
# batch_size = configs['training']['batch_size'],
# normalise = configs['data']['normalise']
# ),
# epochs = configs['training']['epochs'],
# batch_size = configs['training']['batch_size'],
# steps_per_epoch = steps_per_epoch
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
if (choice == "multi"):
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
elif (choice == "seq"):
predictions = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
else:
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):os.makedirs(configs['model']['save_dir'])
model = Model()
my_model = model.build_model(configs)
plot_model(my_model, to_file='output\model.png', show_shapes=True)
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
x, y = data.get_train_data(
configs['data']['sequence_length'],
configs['data']['normalise']
)
print(x.shape)
print(y.shape)
print(configs['training']['batch_size'])
print(configs['model']['save_dir'])
model.train(x,
y,
configs['training']['epochs'],
configs['training']['batch_size'],
configs['model']['save_dir']
)
x_test, y_test = data.get_test_data(
configs['data']['sequence_length'],
configs['data']['normalise']
)
# predictions = model.predict_sequences_multiplt(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequences_full(x_test, configs['data']['sequence_length'])
prediction_point = model.predict_point_by_point(x_test)
# print(prediction_point)
# print(np.array(predictions).shape)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(prediction_point, y_test)
def main():
#load parameters
configs = json.load(open('./data/config.json','r'))
if not os.path.exists(configs['model']['save_dir']):os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data',configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
)
#create RNN model
model=Model()
model.build_model(configs)
#loading trainning data
x,y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
print(x.shape)
print(y.shape)
#training model
model.train(
x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir']
)
#test results
x_test, y_test = data.get_test_data(
seq_len= configs['data']['sequence_length'],
normalise=configs['data']['normalise'],
)
#results visualization
predictions_multiseq = model.predict_sequences_multiple(x_test,configs['data']['sequence_length'],configs['data']['sequence_length'])
predictions_pointbypoint=model.predict_point_by_point(x_test)
plot_results_multiple(predictions_multiseq,y_test,configs['data']['sequence_length'])
plot_results(predictions_pointbypoint,y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
)
model = Model()
model.build_model(configs)
# get train data
x, y = data.get_train_data()
#x=x.squeeze()
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# # out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# batch_size=configs['training']['batch_size'],
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
# testing model
x_test, y_test = data.get_test_data()
#x_test=x_test.squeeze()
predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
def main():
#读取所需参数
configs = json.load(open('config_2.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
#读取数据
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
#创建RNN模型
model = Model()
mymodel = model.build_model(configs)
plot_model(mymodel, to_file='model.png', show_shapes=True)
#加载训练数据
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
print(x.shape)
print(y.shape)
#训练模型
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
#测试结果
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#展示测试效果
predictions_multiseq = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
predictions_pointbypoint = model.predict_point_by_point(x_test, debug=True)
plot_results_multiple(predictions_multiseq, y_test,
configs['data']['sequence_length'])
plot_results(predictions_pointbypoint, y_test)
def main():
configs = json.load(open('config.json', 'r'))
model = Model()
model.load_model("./saved_models/model2.h5")
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
def main_plot():
configs = json.load(open(config_file, 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
normalise_meth=configs['data']['normalise'])
x, y = data.get_test_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
model = Model()
global newest_model
if newest_model:
model_way = newest_model
else:
model_way = '/home/bf/Documents/Projects/helpplay/HelpPlay/train/LSTM-Neural-Network-for-Time-Series-Prediction/saved_models/10062019-163648-e40.h5'
model.load_model(model_way)
print(model.model.evaluate(x, y))
pre_y = model.predict_point_by_point(x)
print(x)
plot_results(pre_y, y)
model_id = configs['model']['model_id']
save_dir = configs['model']['save_dir']
dataloader = DataLoader()
x_scaler_filename = save_dir + "/" + model_id + "-x.scaler"
y_scaler_filename = save_dir + "/" + model_id + "-y.scaler"
dataloader.restore_scalers(x_scaler_filename, y_scaler_filename)
filename = os.path.join('data', configs['data']['filename'])
dataframe = pandas.read_csv(filename, sep=',', encoding='utf-8')
dataframe.index.name = 'fecha'
x_data = dataframe.get(configs['data']['x_cols'], ).values
in_seq_len = configs['data']['input_sequence_length']
x_data = x_data[:, :] # pick three sequences to make predictions
input_data = dataloader.prepare_input_data(x_data, in_seq_len)
print("Input vector shape: " + str(x_data.shape))
model_filename = sys.argv[2]
model = Model(configs['data']['output_mode'])
model.load_model(filepath=model_filename)
print("Plotting predictions point by point on validation set")
predictions = model.predict_point_by_point(input_data)
print(predictions.shape)
unscaled_predictions = dataloader.recompose_results(predictions[:, 0, :],
side="y")
plot_results(unscaled_predictions,
x_data[configs['data']['input_sequence_length']:, :])
def main():
configs = json.load(open('configcrops.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# Yogyakarta: Kulon progo, bantul, gunung kidul, sleman, DIY
# Jawa Barat: Bandung, Tasikmalaya, Majalengka, Cirebon, Kuningan, Garut, Sumedang, Cianjut, Subang, Purwakarta, Indramayu
# Ciamis, Sukabumi, Bogor, Bekasi, Karawang
# # out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
# # save_dir = configs['model']['save_dir']
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# print(x_test)
# print(y_test)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions_point = model.predict_point_by_point(x_test)
print(len(predictions_point))
plot_results(predictions_point, y_test)
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
# predictions_full = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# plot_results(predictions_full, y_test)
groundtrue = data._groundtruths(1)
groundtrue = (groundtrue.ravel())
print(len(groundtrue))
RMSElist = []
for i in range(len(groundtrue)):
errorrate = groundtrue[i] - predictions_point[i]
hasilkuadrat = errorrate * errorrate
RMSElist.append(hasilkuadrat)
RMSE = sum(RMSElist) / (len(predictions_point) - 2)
RMSE = RMSE**(1 / 2)
print(RMSE)
getdataforecast = data._forecasting(5, 1)
total_prediksi = 5
takefrom = 5
forecast_result = model.forecast(total_prediksi, getdataforecast, takefrom)
# print(forecast_result[0])
# forecast_result=np.append(forecast_result,[0.0])
# print(forecast_result)
n_steps = 8
# split into samples
X, y = split_sequence(forecast_result, n_steps)
# reshape from [samples, timesteps] into [samples, timesteps, features]
n_features = 1
# print(X)
X = X.reshape((X.shape[0], X.shape[1], n_features))
# define model
model = Sequential()
model.add(LSTM(50, activation='relu', input_shape=(n_steps, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit(X, y, epochs=200, verbose=0)
# demonstrate prediction
for j in range(total_prediksi):
getxlastnumber = array(forecast_result[(-n_steps - 1):-1])
x_input = getxlastnumber
# print(x_input)
x_input = x_input.reshape((1, n_steps, n_features))
yhat = model.predict(x_input, verbose=0)
# print(yhat[0][0])
forecast_result = np.append(forecast_result, yhat[0])
# prediction_point=np.append(prediction_point,yhat[0])
plot_results_onlypredicted(forecast_result)
def main():
configs = json.load(open('configcrops.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
#filename1 can be changed into filename (see the configcrops.json)
namaefile = configs['data']['filename1']
with open(namaefile, 'r') as dataframe:
hasil = json.load(dataframe)
# print(hasil)
temp = []
listhasil = []
for key, value in hasil.items():
temp = [key, value]
listhasil.append(temp)
listkota = [
'Kulon Progo', 'Bantul', 'Gunung Kidul', 'Sleman', 'DIY', 'Bandung',
'Tasikmalaya', 'Majalengka', 'Cirebon', 'Kuningan', 'Garut',
'Sumedang', 'Cianjur', 'Subang', 'Purwakarta', 'Indramayu', 'Ciamis',
'Sukabumi', 'Bogor', 'Bekasi', 'Karawang'
]
kodekota = [
'KLP', 'BTL', 'GKD', 'SLM', 'DIY', 'BD', 'TKM', 'MJK', 'CRB', 'KNG',
'GRT', 'SMD', 'CJR', 'SBG', 'PWK', 'IDY', 'CMS', 'SKB', 'BGR', 'BKS',
'KRW'
]
listtahun = []
for i in range(1961, 2015):
listtahun.append(str(i))
data = []
#listhasil
#=[["Kulon Progo",{data tahun dan crops},"DIY",{data tahun dan crops}]
datacrops = []
datalengkapcrops = []
datalengkaptahun = []
datatahun_semuadaerah = []
#Variabel untuk tampung data csv
kota_untuk_csv = [] #pembuatan kolom kota pada csv
kode_untuk_csv = [] #pembuatan kolom kode untuk csv
tahun_untuk_csv = [] #pembuatan kolom tahun untuk csv
crops_untuk_csv = [] #pembuatan kolom crops untuk csv
RMSE_untuk_csv = []
semua_data_csv = [
] #untuk menampung data kota,kode,tahun, dan crops beserta rmse pada sebaris (tidak digunakan #cadangan)
#Pengulangan compiling per kota
for j in range(len(listkota)):
if (len(listhasil[j][1]) != len(listtahun)):
jlhprediksi = len(listtahun) - len(
hasil[listkota[j]]) + (6) - 1 #prediksi sampai 2020
else:
jlhprediksi = (6) - 1 #prediksi sampai 2020
datatahun_crops = listhasil[j][1] #dapat data json crops dan tahun
datatahun = list(
datatahun_crops.keys()) #dapat data tahun pada satu daerah
datatahunint = [int(x) for x in datatahun] #konversi ke integer
arraytahun = np.array(datatahunint) #dibuat jadi array
sorttahun = np.sort(arraytahun) #sort dalam bentuk array
datatahun_daerah = list(sorttahun) #buat lagi jadi list
datalengkaptahun.append(datatahun_daerah)
for n in range(len(
listhasil[j][1])): #listhasil[j][1] = data tahun dan crops
datacrops.append(float(listhasil[j][1][str(datatahun_daerah[n])]))
datalengkapcrops.append(datacrops)
datacrops = []
# print(datalengkapcrops)
# print(datalengkaptahun)
listcrops_daerah = []
hasillistcrops_daerah = []
for i in range(len(listkota)):
for j in range(len(datalengkapcrops[i])):
listcrops_daerah.append([datalengkapcrops[i][j]
]) #pecah per tahun crops dalam satu list
hasillistcrops_daerah.append(listcrops_daerah)
listcrops_daerah = []
# for i in range(len(listkota)):
# arraycrops_semua=np.array(hasillistcrops_daerah[i])
# print(arraycrops_semua[0:20])
arraycrops_semua = np.array(hasillistcrops_daerah)
for i in range(len(listkota)):
data = DataLoader(np.array(arraycrops_semua[i]),
configs['data']['train_test_split'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# Yogyakarta: Kulon progo, bantul, gunung kidul, sleman, DIY
# Jawa Barat: Bandung, Tasikmalaya, Majalengka, Cirebon, Kuningan, Garut, Sumedang, Cianjut, Subang, Purwakarta, Indramayu
# Ciamis, Sukabumi, Bogor, Bekasi, Karawang
# # out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
# # save_dir = configs['model']['save_dir']
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# print(x_test)
# print(y_test)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
predictions_point = model.predict_point_by_point(x_test)
print(len(predictions_point))
for ulang in range(len(datalengkaptahun[i]) - len(predictions_point)):
datalengkaptahun[i].remove(
datalengkaptahun[i]
[ulang]) #for equality number of ground truth and prediction
# Use the plot when you want to see the data graphically
# plot_results(predictions_point, y_test,datalengkaptahun[i],listkota[i])
groundtrue = data._groundtruths(1)
groundtrue = (groundtrue.ravel())
# print(len(groundtrue))
#Measure the RMSE
RMSElist = []
for k in range(len(predictions_point)):
errorrate = groundtrue[k + ulang] - predictions_point[k]
hasilkuadrat = errorrate * errorrate
RMSElist.append(hasilkuadrat)
RMSE = sum(RMSElist) / (len(predictions_point))
RMSE = RMSE**(1 / 2)
# print(RMSE)
getdataforecast = data._forecasting(jlhprediksi, jlhprediksi)
# print(len(getdataforecast))
total_prediksi = jlhprediksi
takefrom = jlhprediksi
forecast_result = model.forecast(total_prediksi, getdataforecast,
takefrom)
# print(len(forecast_result))
# print(forecast_result[0])
# forecast_result=np.append(forecast_result,[0.0])
# print(forecast_result)
n_steps = 8
# split into samples
X, y = split_sequence(forecast_result, n_steps)
# reshape from [samples, timesteps] into [samples, timesteps, features]
n_features = 1
# print(X)
X = X.reshape((X.shape[0], X.shape[1], n_features))
# define model
model = Sequential()
model.add(
LSTM(50, activation='relu', input_shape=(n_steps, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit(X, y, epochs=200, verbose=0)
#make the number of predictions is equal to the number of the ground truth
hasilprediksi = []
hasilprediksi.append(groundtrue[-(ulang + 1):-ulang])
hasilprediksi.append(groundtrue[-ulang:])
for j in range(total_prediksi):
getxlastnumber = array(forecast_result[(-n_steps - 1):-1])
x_input = getxlastnumber
# print(x_input)
x_input = x_input.reshape((1, n_steps, n_features))
yhat = model.predict(x_input, verbose=0)
# print(yhat[0][0])
hasilprediksi.append(yhat[0]) #untuk dikirimkan ke json
forecast_result = np.append(forecast_result,
yhat[0]) #untuk training forecast
groundtrue = np.append(groundtrue,
yhat[0]) #untuk plotting ke grafik
# print(len(groundtrue))
# prediction_point=np.append(prediction_point,yhat[0])
# print(hasilprediksi) #hasilprediksi dalam bentuk array, hasilprediksi[0] dalam bntk list, hasilprediksi[0][0] dalam bentuk skalar
semuatahun = datalengkaptahun[i]
tahunbaru = []
terakhirtahun = datalengkaptahun[i][len(datalengkaptahun[i]) - 1]
rangetahun_input = len(groundtrue) - len(datalengkaptahun[i])
# print(rangetahun_input)
if (len(datalengkaptahun[i]) < len(groundtrue)):
for z in range(rangetahun_input):
semuatahun.append(terakhirtahun) #untuk grafik
tahunbaru.append(terakhirtahun) #untuk dikirimkan ke json
terakhirtahun = terakhirtahun + 1
# print(tahunbaru)
# Use the plot when you want to see the data graphically
# plot_results_onlypredicted(semuatahun,groundtrue,listkota[i])
#To check the length of ground true is equal to the datalengkaptahun[i] or the number of years record at specific Entity
# print(len(groundtrue))
# print(len(datalengkaptahun[i]))
# semuahasil_csv=[]
# csv_data_kota=duplikathasil.get(column).values[:]
#To record all data into LIST to make CSV
for jlh in range(len(semuatahun)):
kota_untuk_csv.append(listkota[i])
kode_untuk_csv.append(kodekota[i])
RMSE_untuk_csv.append(RMSE[0])
tahun_untuk_csv.append(semuatahun[jlh])
crops_untuk_csv.append(groundtrue[jlh])
#Alternative solution for csv
# for jlh in range(len(datalengkapcrops[i])):
# kota_untuk_csv.append(listkota[i])
# kode_untuk_csv.append(kodekota[i])
# # tahun_untuk_csv.append(datalengkaptahun[i][jlh])
# # crops_untuk_csv.append(datalengkapcrops[i][jlh])
# RMSE_untuk_csv.append(RMSE[0])
# for jlh in range (rangetahun_input):
# kota_untuk_csv.append(listkota[i])
# kode_untuk_csv.append(kodekota[i])
# # tahun_untuk_csv.append(tahunbaru[jlh])
# # crops_untuk_csv.append(hasilprediksi[jlh][0])
# RMSE_untuk_csv.append(RMSE[0])
HasilCSV = {
'Entity': kota_untuk_csv,
'Code': kode_untuk_csv,
'Year': tahun_untuk_csv,
' crop(tonnes per hectare)': crops_untuk_csv,
'RMSE': RMSE_untuk_csv
}
df = DataFrame(HasilCSV,
columns=[
'Entity', 'Code', 'Year',
' crop(tonnes per hectare)', 'RMSE'
])
print(df)
filebaca_csv = configs["data"]["newcsv"]
filebaca_csv1 = configs["data"]["newcsv1"]
#name of data export can be changed through configcrops.json (variable filebaca_csv and filebacacsv1 only for explanation)
export_csv = df.to_csv(
r'/home/biovick/Downloads/tkte/sudiro/Forecasting-and-Predicting Crops into Visualization/data/newTomatov2.csv',
index=False)
def main():
configs = json.load(open('point_to_point_similar_sinewave.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
# out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
predictions = predictions.reshape(-1, 1)
normalized_test = data.get_normalized_test()
normalized_test = np.delete(normalized_test, [j for j in range(len(predictions), len(normalized_test))], axis=0)
my_normalized_test = np.delete(normalized_test, [0], axis=1)
# my_normalized_test = np.delete(my_normalized_test, [j for j in range(len(predictions), len(my_normalized_test))], axis=0)
final_data = np.hstack((predictions, my_normalized_test))
actual_predictions = data.inverse_data(final_data)
predictions = actual_predictions[:, 0]
actual_test = data.inverse_data(normalized_test)
y_test = actual_test[:, 0]
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test)
mse = mean_squared_error(y_test, predictions)
print("Mean Squared Error: " + str(mse))
print("Root Mean Squared Error: " + str(math.sqrt(mse)))
mae = mean_absolute_error(y_test, predictions)
print("Mean Absolute Error: " + str(mae))
r2 = r2_score(y_test, predictions)
print("R Squared Error: " + str(r2))
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
lossesMINE = []
lossesKERAS = []
# for day_prediction in [1, 2, 3, 4, 5, 10, 50]:
day_prediction = 10
print("Predicting %i days..." % day_prediction)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'],
day_pred=day_prediction)
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'])
# out-of memory generative training
# steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise'],
# day_pred=day_prediction
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'],
day_pred=day_prediction)
# print(x_test.shape)
# print(len(data.denormalization_vals))
# print(y_test.shape)
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
#
# y_test_unormalized = np.zeros((y_test.shape[0], ))
# prediction_unormalized = []
#
# for i in range(4):
# for j in range(int(configs['data']['sequence_length']) - 10):
# y_test_unormalized[j*(i+1)] = (y_test[j] + 1)*data.data_test[i*int(configs['data']['sequence_length']), 0]
# prediction_unormalized.append((predictions[j*(i+1)] + 1)*data.data_test[i*int(configs['data']['sequence_length']), 0])
npPredictions = np.asarray(predictions)
# print(type(npPredictions))
# print(type(y_test))
# print(npPredictions.shape)
# print(y_test.shape)
loss = 0
for i in range(len(npPredictions)):
loss += (npPredictions[i] - y_test[i])**2
print(loss)
keras_loss = model.model.evaluate(x_test, y_test)
print(keras_loss)
lossesMINE.append(loss)
lossesKERAS.append(keras_loss)
#plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
real_y = np.reshape(y_test, (y_test.shape[0], )) * np.asarray(
data.denormalization_vals) + np.asarray(data.denormalization_vals)
real_pred = predictions * np.asarray(
data.denormalization_vals) + np.asarray(data.denormalization_vals)
# print(real_y.shape)
# print(real_pred.shape)
data.denormalization_vals = []
#plot_results(predictions, y_test)
plot_results(real_pred, real_y)
print(lossesMINE)
print(lossesKERAS)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
# model.build_model(configs)
model.load_model("saved_models/dow_30_50%.h5")
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
print("x_test.shape")
print(x_test.shape)
predictions = model.predict_point_by_point(x_test)
########################################################################
from sklearn.metrics import mean_squared_error
# loss_final = mean_squared_error(predictions, y_test)
# print("Testing Loss = " + str(loss_final))
########################################################################
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
print(predictions.shape)
print(y_test.shape)
m = pd.DataFrame(predictions)
n = pd.DataFrame(y_test)
m.to_csv("predictions.csv")
n.to_csv("y_test.csv")
p = 0
t = 0
t_1 = 0
count = 0
for a in range(len(predictions)):
if (a == 0):
t_1 = y_test[a]
continue
'''
1 1 1 1 1 1 1 1 1
1 1 1 1 1 1 1 1 1
'''
p = predictions[a]
t = y_test[a]
match = (t - t_1) * (p - t_1)
if (match > 0):
count += 1
t_1 = t
print("Good prediction rate = " + str(count / len(predictions)))
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
if not os.path.exists(configs['data']['data picture save dir']):
os.makedirs(configs['data']['data picture save dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'], configs['data']['id'])
model = Model()
model.build_model(configs)
# x, y = data.get_train_data(
# seq_len=configs['data']['sequence_length'],
# normalise=configs['data']['normalise']
# )
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
sess = backend.get_session()
rmsee = backend.mean(rmse(y_test, predictions), axis=0)
msee = backend.mean(mse(y_test, predictions), axis=0)
with sess.as_default():
mse_val = msee.eval()
rmse_val = rmsee.eval()
print("mse:", mse_val)
print("rmse:", rmse_val)
#plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
plot_results(predictions, y_test, configs['data']['data picture save dir'],
configs['data']['id'])
with open("note.txt", 'a+') as f:
f.write(
'\n%s-e%s.h5:\n' %
(dt.datetime.now().strftime('%m%d-%H%M%S'), configs['data']['id']))
f.write("data split:%f\n" % configs["data"]["train_test_split"])
f.write("epochs:%d\n" % configs["training"]["epochs"])
f.write("batch size:%d\n" % configs["training"]["batch_size"])
f.write("mse:%f\n" % mse_val)
f.write("rmse:%f\n" % rmse_val)
f.write("notes:%s\n" % configs['data']['note'])
# start training PBP model
model_PBP.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
# start predicting
predictions_PBP = model_PBP.predict_point_by_point(x_test)
# save the model
pickle.dump(model_PBP, open("model_PBP_baseline.pkl", 'wb'))
# same for multi-sequence predicting
model_MS = Model()
model_MS.build_model(configs)
model_MS.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
return predicted
def predict_sequence_full(self, data, window_size):
#Shift the window by 1 new prediction each time, re-run predictions on new window
curr_frame = data[0]
predicted = []
for i in range(len(data)):
predicted.append(self.model.predict(curr_frame[newaxis, :, :])[0, 0])
curr_frame = curr_frame[1:]
curr_frame = np.insert(curr_frame, [window_size - 2],
predicted[-1],
axis=0)
return predicted
def plot_results(predicted_data, true_data):
fig = plt.figure(facecolor='white')
ax = fig.add_subplot(111)
ax.plot(true_data, label='True Data')
plt.plot(predicted_data, label='Prediction')
plt.legend()
plt.show()
predictions_pointbypoint = model.predict_point_by_point(x_test)
plot_results(predictions_pointbypoint, y_test)
predictions_fullseq = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
plot_results(predictions_fullseq, y_test)
