def train_epoc(trn, model, criterion, optimizer, scheduler):
# Train the model
model.train()
train_loss = 0
train_acc = 0
for label, data_x, data_len in tqdm(trn):
#print("label",label)
#print("data_x",data_x)
#print("data_len",data_len)
optimizer.zero_grad()
label = label.to(device)
data_len = data_len.to(device)
for i in range(len(data_x)):
#data_x[i]=rnn_utils.pack_padded_sequence(data_x[i].to(device),data_len,batch_first=True,enforce_sorted=False)
data_x[i] = data_x[i].to(device)
output = model(data_x, data_len)
loss = criterion(output, label)
train_loss += loss.item()
loss.backward()
optimizer.step()
train_acc += (output.argmax(1) == label).sum().item()
# Adjust the learning rate train_acc
scheduler.step()
return train_loss, train_acc
def stocks_prediction(prediction_days, stock):
print(f"Executing prediction for {stock}")
yf.pdr_override()
now = time.strftime('%Y-%m-%d', time.localtime(time.time() + 86400))
data = pdr.get_data_yahoo(stock, start=config.HISTORY_START_DATE, end=now)
db = database.Database()
db.use("taurus")
db.panda_write("taurus", data, stock)
# Save Simple Prediction to DB
prediction_results = prediction.simple_prediction(prediction_days, data)
simple_data = _save("Prediction", prediction_results, stock)
model = lstm.model(prediction_days, data)
# Save LSTM Prediction to DB
lstm_results = lstm.predict(model[0], model[1], model[2], model[3],
model[4])
lstm_data = _save("LSTMPrediction", lstm_results, stock)
# Save Root Deviation to DB
rmse_results = lstm.root_deviation(lstm_results, model[0])
# The var writing is on the opposite way so I can generate a query on influxdb for all the deviations.
_save(stock, rmse_results, "Deviation")
return simple_data, lstm_data
def val_epoc(tst, model, criterion):
model.eval()
loss = 0
acc = 0
for label, data_x, data_len in tst:
label = label.to(device)
data_len = data_len.to(device)
for i in range(len(data_x)):
#data_x[i]=rnn_utils.pack_padded_sequence(data_x[i].to(device),data_len,batch_first=True,enforce_sorted=False)
data_x[i] = data_x[i].to(device)
with torch.no_grad():
output = model(data_x, data_len)
loss = criterion(output, label)
loss += loss.item()
acc += (output.argmax(1) == label).sum().item()
return loss, acc
scaler, raw_data_scaled = scale(data_raw, split_pct) # Scale all of the data
data_scaled = timeseries_to_supervised(
raw_data_scaled, timesteps) # turn it into timeshifted data
x_train, y_train, x_test, y_test = split_and_reshape(
data_scaled, raw_data_scaled, split_pct, timesteps,
data_dim) # Uses shifted data for x and original data for y
train_losses_in_experiments = np.zeros([len(experiment_neurons), epochs])
test_losses_in_experiments = np.zeros(len(experiment_neurons))
experiment = 0
seed = 0
for neurons in experiment_neurons:
print('Experiment %d Beginning.' % (experiment))
this_model = model(neurons, timesteps, data_dim)
train_loss_list = np.zeros([runs, epochs])
test_loss_list = np.zeros(runs)
for r in range(runs):
np.random.seed(seed)
model_history = this_model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
shuffle=False)
train_loss = model_history.history['loss']
train_loss_list[r, :] = train_loss
def main(ticker, epoch, timeframe, indictors):
stock = pd.DataFrame(index=normalizedCloseddf.index)
#Extract the ticker
#Stick all indicators in the combination which fits the company to dataframe "stock"
for indictor in indictors:
if str(indictor) == str(IXIC):
stock = stock.join(IXIC)
stock = stock.join(indictor.filter(regex=ticker))
#startDate is marked the date, user wants to start the dataset.
startDate = stock.index.get_loc('2005-01-01')
stock = stock.drop(stock.index[0:startDate])
print(stock.head(5))
#this function will load the first 20% of data as testing and 80% till the latest as training data
X_train, y_train, X_test, y_test = lstm.load_dataforward(
stock[::-1], timeframe)
#this function will load the last 20% of data as testing and 80% from the beginning as training data
#X_train, y_train, X_test, y_test = lstm.load_data(stock[::-1], 30)
print("X_train", X_train.shape)
print("y_train", y_train.shape)
print("X_test", X_test.shape)
print("y_test", y_test.shape)
#tbcallback will generate a graph which can show back in tensorboard
tbCallBack = keras.callbacks.TensorBoard(log_dir="./Graph",
histogram_freq=0,
write_graph=True,
write_images=True)
#model will have three variables, the first variable is number of indicators, the second variable is the timeframe
model = lstm.model([len(indictors), timeframe])
model.fit(X_train,
y_train,
batch_size=150,
nb_epoch=epoch,
validation_split=0.2,
callbacks=[tbCallBack],
verbose=1)
model.save(ticker + '.h5')
trainScore = model.evaluate(X_train, y_train, verbose=0)
print('Train Score: %.2f MSE (%.2f RMSE)' %
(trainScore[0], math.sqrt(trainScore[0])))
testScore = model.evaluate(X_test, y_test, verbose=0)
print('Test Score: %.2f MSE (%.2f RMSE)' %
(testScore[0], math.sqrt(testScore[0])))
prediction = model.predict(X_test)
prediction = prediction[::-1]
y_test = y_test[::-1]
peakperiod = 5
peaksdetect = peakdetect(prediction, lookahead=peakperiod)
peaksdetect = np.array(peaksdetect)
peaks = peaksdetect[0]
toughs = peaksdetect[1]
peakpoints = [item[1] for item in peaks]
peakdays = [item[0] for item in peaks]
toughpoints = [item[1] for item in toughs]
toughdays = [item[0] for item in toughs]
plt.figure(figsize=(18, 7))
plt.plot(prediction, color='red', label='prediction')
plt.scatter(peakdays, peakpoints, color='red', marker='x')
plt.scatter(toughdays, toughpoints, color='green', marker='o')
plt.plot(y_test, color='blue', label='y_test')
plt.legend(loc='upper left')
plt.tight_layout()
plt.subplots_adjust(top=0.95, left=0.08, bottom=0.08)
strIndictors = ''.join(str(e) for e in indictors)
title = epoch, " epoch ", timeframe, ' timeframe', ticker, list(stock)
plt.title(title)
plt.xlabel("Time")
plt.ylabel("Price")
listIndicators = list(stock)
epochstr = str(epoch)
timeframestr = str(timeframe)
peakperiod = str(peakperiod)
strIndicators = ''.join(listIndicators)
csvname = epochstr + ' e' + timeframestr + ' t' + strIndicators + ' peak ' + peakperiod + '.csv'
np.savetxt(csvname, np.c_[prediction, y_test], delimiter=",")
filename = epochstr + ' e' + timeframestr + ' t' + strIndicators + ' peak ' + peakperiod + '.png'
plt.savefig(filename)
if __name__ == '__main__':
print(sys.argv)
if len(sys.argv) == 1:
print(' \"python run_lstm.py -train\" train lstm\n '
' \"python run_lstm.py -rescore\" rescore nbest\n'
' \"python run_lstm.py -wer\" compute WER')
fres = wb.FRes('result.txt')
for tsize in [1, 2, 4]:
tskdir = '{}/'.format(tsize)
print(tskdir)
workdir = tskdir + 'lstmlm/'
bindir = '../../tools/lstm'
model = lstm.model(bindir, workdir)
datas = data(tskdir)
hidden = 250
dropout = 0
epoch = 10
gpu = 1
write_model = workdir + 'h{}_dropout{}_epoch{}.lstm'.format(
hidden, dropout, epoch)
write_name = '{}:LSTM:h{}d{}epoch{}'.format(tsize, hidden, dropout,
epoch)
config = '-hidden {} -dropout {} -epoch {} -gpu {}'.format(
hidden, dropout, epoch, gpu)
if '-train' in sys.argv or '-all' in sys.argv:
if os.path.exists(write_model):