def train_model(stock_name,
x_train,
y_train,
x_test,
y_test,
sc_close,
unroll_length=50,
retrain=False):
batch_size = 32
epochs = 10
if retrain:
model = lstm.lstm_model(x_train.shape[-1],
output_dim=unroll_length,
return_sequences=True)
model.compile(loss='mean_squared_error', optimizer='adam')
#start = time.time()
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=2,
validation_split=0.05)
model.save(os.getcwd() + "/data/" + stock_name + ".h5")
else:
if not os.path.isfile(os.getcwd() + "/data/" + stock_name + ".h5"):
model = lstm.lstm_model(x_train.shape[-1],
output_dim=unroll_length,
return_sequences=True)
model.compile(loss='mean_squared_error', optimizer='adam')
#start = time.time()
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=2,
validation_split=0.05)
model.save(os.getcwd() + "/data/" + stock_name + ".h5")
#print('training_time', time.time() - start)
else:
model = load_model(os.getcwd() + "/data/" + stock_name + ".h5")
prediction = model.predict(x_test, batch_size=batch_size)
prediction = sc_close.inverse_transform(prediction)
#print(prediction)
y_test = sc_close.inverse_transform(y_test)
#vs.plot_lstm_prediction(y_test, prediction)
return model, prediction
def __init__(self, fakeways=[], arp_for_unknowns=False, wide=False):
# These are "fake gateways" -- we'll answer ARPs for them with MAC
# of the switch they're connected to.
self.fakeways = set(fakeways)
# If True, we create "wide" matches. Otherwise, we create "narrow"
# (exact) matches.
self.wide = wide
# If this is true and we see a packet for an unknown
# host, we'll ARP for it.
self.arp_for_unknowns = arp_for_unknowns
# (dpid,IP) -> expire_time
# We use this to keep from spamming ARPs
self.outstanding_arps = {}
# (dpid,IP) -> [(expire_time,buffer_id,in_port), ...]
# These are buffers we've gotten at this datapath for this IP which
# we can't deliver because we don't know where they go.
self.lost_buffers = {}
# For each switch, we map IP addresses to Entries
self.arpTable = {}
# This timer handles expiring stuff
self._expire_timer = Timer(5, self._handle_expiration, recurring=True)
core.listen_to_dependencies(self)
### edited ###
print 'starting lstm switch'
self.startTime = time.time()
self.lstm = lstm.lstm_model()
def setUpClass(cls):
"""
initialize the lstm model
"""
cls.n_layers = 5
cls.batch_size = 20
cls.l = 10 # length of recursive chain
np.random.seed(237)
# the input and output dimensions of each layer
cls.ns = np.array(
[np.random.randint(1, 5) for i in range(cls.n_layers)])
cls.ms = np.concatenate((cls.ns[1:], [np.random.randint(1, 5)]))
# the model
cls.model = lstm_model(cls.ns, cls.ms, seed=102)
# compile the network
start = time.clock()
cls.model.compile_network()
cls.model_network_compile_time = time.clock() - start
# save the weight matrices
cls.Ws0 = cls.model.Ws.get_value()
cls.Us0 = cls.model.Us.get_value()
cls.bs0 = cls.model.bs.get_value()
# tolerance for numerical optimization
cls.tol = 1E-2
# learning rate
cls.eta = 1.
# a single layer model
cls.nn = 3
cls.mm = 5
# one layer model for testing against by-hand gradients
cls.one_layer_model = lstm_model(np.array([cls.nn]),
np.array([cls.mm]),
seed=1434)
def main():
clearLogFolder()
load_russian_alphabet(ALPHABET_PATH, alphabet)
if USE_SUPERTYPES:
splitToSupertypes(supertypes)
g = Graph()
g = g.Read(data_in)
get_all_seqs(g)
if len(all_seq) > 0:
print("Parsingsequences")
X, y = processSequences(all_seq, TIMESTEPS)
print("Merging sequences")
Xm, ym = mergeSeqs(X, y)
print("Create regressor")
regressor = learn.SKCompat(
learn.Estimator(model_fn=lstm_model(TIMESTEPS,
RNN_LAYERS,
DENSE_LAYERS,
optimizer="Adam"),
model_dir=LOG_DIR))
# create a lstm instance and validation monitor
validation_monitor = learn.monitors.ValidationMonitor(
Xm['val'],
ym['val'],
every_n_steps=PRINT_STEPS,
early_stopping_rounds=1000)
print("fit regressor")
regressor.fit(Xm['train'],
ym['train'],
monitors=[validation_monitor],
batch_size=BATCH_SIZE,
steps=TRAINING_STEPS)
print("predicting")
predicted = regressor.predict(Xm['test'])
# rmse = np.sqrt(((predicted - ym['test']) ** 2).mean(axis=0))
score = mean_squared_error(predicted, ym['test'])
hited = hitpoint(predicted, ym['test'])
print("MSE: %f" % score)
print("hitpoint:", hited)
# save dataset
os.mkdir(data_path)
pickle.dump(
X,
open(
data_path + "/x_set" + str(lstm.IN_TIMESTEPS) +
str(lstm.OUT_TIMESTEPS_RANGE[-1]) + ".pkl", "wb"))
pickle.dump(
Y,
open(
data_path + "/y_set" + str(lstm.IN_TIMESTEPS) +
str(lstm.OUT_TIMESTEPS_RANGE[-1]) + ".pkl", "wb"))
print("Save data successfully!")
## build the lstm model
model_fn = lstm_model()
config = tf.contrib.learn.RunConfig(log_step_count_steps=200,
save_checkpoints_steps=VALIDATION_STEPS //
2)
estimator = learn.Estimator(model_fn=model_fn,
model_dir=LOG_DIR,
config=config)
regressor = learn.SKCompat(estimator)
## create a validation monitor
validation_monitor = learn.monitors.ValidationMonitor(
X['val'], Y['val'], every_n_steps=VALIDATION_STEPS)
## fit the train dataset
# TRAINING_STEPS = 1
from data_processing import load_csvdata
import xlrd
import random
import math
import xlrd
LOG_DIR = 'C:/Users/chak282/Downloads'
TIMESTEPS = 3
RNN_LAYERS = [{'num_units': 5}]
DENSE_LAYERS = None
TRAINING_STEPS = 10000
PRINT_STEPS = TRAINING_STEPS / 10
BATCH_SIZE = 100
regressor = learn.SKCompat(
learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS),
model_dir=LOG_DIR))
workbook = xlrd.open_workbook('RNN_data.xlsx')
sheet1 = workbook.sheet_by_name('RNN_data')
data = np.zeros(shape=sheet1.nrows)
for index1 in range(0, sheet1.nrows):
data[index1] = sheet1.cell_value(index1, 1)
X, y = load_csvdata(data, TIMESTEPS, seperate=False)
# create a lstm instance and validation monitor
validation_monitor = learn.monitors.ValidationMonitor(
X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000)
# print(X['train'])
from tensorflow.contrib import learn
from sklearn.metrics import mean_squared_error
from lstm import lstm_model
from data_processing import generate_data
tf.logging.set_verbosity(tf.logging.INFO)
LOG_DIR = './ops_logs/sin'
TIMESTEPS = 3
RNN_LAYERS = [{'num_units': 5}]
DENSE_LAYERS = None
TRAINING_STEPS = 10000
PRINT_STEPS = TRAINING_STEPS / 10
BATCH_SIZE = 100
regressor = learn.SKCompat(
learn.Estimator(
model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS),
model_dir=LOG_DIR))
X, y = generate_data(
np.sin,
np.linspace(0, 100, 10000, dtype=np.float32),
TIMESTEPS,
seperate=False)
# create a lstm instance and validation monitor
validation_monitor = learn.monitors.ValidationMonitor(X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000)
print(X['train'])
print(y['train'])
regressor.fit(
X['train'],
y['train'],
from sklearn.metrics import mean_squared_error
from lstm import generate_data, lstm_model
import logging
logging.basicConfig(level=logging.DEBUG)
LOG_DIR = '/tmp/m/ops_logs/6/ts'
TIMESTEPS = 13
RNN_LAYERS = [{'num_units': 32},{'num_units': 64},{'num_units': 64}]
DENSE_LAYERS = None
TRAINING_STEPS = 40000
PRINT_STEPS = TRAINING_STEPS / 100
BATCH_SIZE = 20
regressor = learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS))
X, y = generate_data(np.sin, np.linspace(0, 100, 10000, dtype=np.float32), TIMESTEPS, seperate=False)
'''
tushare data
'''
import tushare as ts
tdata = ts.get_hist_data('600848',ktype='d')
'''
fdata = []
fdata.append([w for w in tdata.open])
fdata.append([w for w in tdata.close])
fdata.append([w for w in tdata.low])
fdata.append([w for w in tdata.high])
TIMESTEPS = 20
RNN_LAYERS = [{'num_units': 5}]
DENSE_LAYERS = [10, 10]
TRAINING_STEPS = 100000
BATCH_SIZE = 100
PRINT_STEPS = TRAINING_STEPS / 100
# We want to predict the Closing price. Not too much data
close = pred.data.Close.ix[(len(pred.props) - 252 * 2):]
# Split the data into test and train
X, y = load_csvdata(close, TIMESTEPS, seperate=False)
regressor = learn.Estimator(model_fn=lstm_model(TIMESTEPS,
RNN_LAYERS,
DENSE_LAYERS,
learning_rate=0.05,
optimizer="Adagrad"),
model_dir=LOG_DIR)
# create a lstm instance and validation monitor
validation_monitor = learn.monitors.ValidationMonitor(
X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000)
# Fit the data to the models
regressor.fit(X['train'],
y['train'],
monitors=[validation_monitor],
batch_size=BATCH_SIZE,
steps=TRAINING_STEPS)
# Make the prediction
realX = realX.astype('float32')
realY = realY.astype('float32')
# test data
tX = range(0, 243)
testX = []
for i in range(152, 182 + 61 - TIMESTEPS):
testX.append([[x] for x in tX[i:i+TIMESTEPS]])
testX = np.array(testX)
testX = testX.astype('float32')
# model
regressor = skflow.TensorFlowEstimator(
model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS), n_classes=0,
verbose=1, steps=TRAINING_STEPS,
optimizer='Adagrad',
learning_rate=0.03,
batch_size=BATCH_SIZE)
regressor.fit(realX, realY, logdir=LOG_DIR)
predict = regressor.predict(testX)
day = [(datetime.date(2015, 9, 1) +
datetime.timedelta(x)).strftime("%Y%m%d") for x in range(60)]
res = pd.DataFrame()
res['artist_id'] = [a1 for x in range(60)]
res['plays'] = predict[1:]
def DL_BuySell(price, days_future, pred,limit=0.0051, days_previous=504, train_split=0.9, periodM = 9,
fastM = 12, slowM = 26, upperS = 80, lowerS = 20, daysS = 3, periodS = 14):
"""
This function takes the previous stock price and makes a LSTM deep learning network based on it to predict
the future stock price. It is bound by a percentage change limit, stokementric oscillator and momentum considerations.
:param price: pandas DataFrame, All technical details. generated from Predictors class
:param days_future: pandas DateTime, The timestamps you want to predict
:param limit: float, the minimum limit for which trades can occur
:param days_previous: int, How many previous days should be simulated
:param train_split: float, Training/Testing split between (0, 1)
:param fastM: Period to calculate the fast mean (Exponential)
:param periodM: MACD Smoothing number of days
:param slowM: Period to calculate the slow mean (Exponential)
:param upperS: Upper bound of the stokementric oscillator
:param lowerS: Lower bound of the stokementric oscillator
:param daysS: Period to calculate the mean (stokementric oscillator)
:param periodS: Number of days to smooth the stokementric oscillator
:return: pandas DataFrame containing Buy and Sell commands.
"""
LOG_DIR = '.opt_logs/lstm_stock'
TIMESTEPS = 20
RNN_LAYERS = [{'num_units': 5}]
DENSE_LAYERS = [10, 10]
TRAINING_STEPS = 100000
BATCH_SIZE = 100
PRINT_STEPS = TRAINING_STEPS / 100
# Split into testing and training data.
data = price.ix[price.index < days_future[0]]
data = data.ix[-1*(days_previous):]
nval = int(round(len(data) * (1 - train_split)))
df_train, df_val, df_test = data.iloc[:nval], data.iloc[nval:], price.ix[days_future]
labels = False
train_x, val_x, test_x = (rnn_data(df_train, TIMESTEPS, labels=labels),
rnn_data(df_val, TIMESTEPS, labels=labels),
rnn_data(df_test, TIMESTEPS, labels=labels))
labels = True
train_y, val_y, test_y = (rnn_data(df_train, TIMESTEPS, labels=labels),
rnn_data(df_val, TIMESTEPS, labels=labels),
rnn_data(df_test, TIMESTEPS, labels=labels))
X, y = dict(train=train_x, val=val_x, test=test_x), dict(train=train_y, val=val_y, test=test_y)
# Train the model
regressor = learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS,
learning_rate=0.05, optimizer="Adagrad"),
model_dir=LOG_DIR)
# create a lstm instance and validation monitor
validation_monitor = learn.monitors.ValidationMonitor(X['val'], y['val'],
every_n_steps=PRINT_STEPS,
early_stopping_rounds=1000)
# Fit the data to the models
regressor.fit(X['train'], y['train'],
monitors=[validation_monitor],
batch_size=BATCH_SIZE,
steps=TRAINING_STEPS)
# Make the prediction
predicted = list(regressor.predict(X['test']))
# Can we make money?
if (np.max(predicted)-np.min(predicted))/np.max(predicted) < limit:
return
index = price.ix[data.index[0]:days_future.index[-1]].index
pred_val = np.concatenate((price.ix[data.index[0]:days_future.index[0]].values,predicted))
C = pd.DataFrame({"Close": pred_val}, index=index)
stok_Sig = pred.stokO_signal(upper=upperS, lower=lowerS, days=daysS, period=periodS, C=C)
MACD_Sig = pred.MACD_signal_line(period=periodM, hist=True, signal=True, fast=fastM, slow=slowM, C=C)
# Where do all the signals overlap
C["regime"] = (stok_Sig + MACD_Sig)
C["regime"].ix[C["regime"] == 2] = 1
C["regime"].ix[C["regime"] == 1] = 0
C["regime"].ix[C["regime"] == 0] = 0
C["regime"].ix[C["regime"] == -1] = 0
C["regime"].ix[C["regime"] == -2] = -1
# We always have to end on a sell signal
CC = C[C.regime != 0].copy()
temp = CC['regime'].ix[-1]
CC['regime'].ix[-1] = 0
CC['signal'] = np.sign(CC['regime'] - CC['regime'].shift(1))
CC['regime'].ix[-1] = temp
# Make a dataframe containing the buy and sell signals for back testing.
moves = pd.concat([
pd.DataFrame({"Price": CC.loc[CC["signal"] == 1, "Close"],
"Regime": CC.loc[CC["signal"] == 1, 'regime'],
"Signal": "Buy"}),
pd.DataFrame({"Price": CC.loc[CC["signal"] == -1, "Close"],
"Regime": CC.loc[CC["signal"] == -1, 'regime'],
"Signal": "Sell"})
])
moves.sort_index(inplace=True)
if moves.ix[0].Regime == -1:
moves = moves.ix[1:]
return moves
from import_dataset import importer
from lstm import lstm_model
import matplotlib.pyplot as plt
if __name__ == '__main__':
df = importer()
#plot
plt.figure(figsize=(16,8))
plt.plot(df['Close'], label='Close Price history')
lstm_model(df)
label_array.shape
# Next, we build a deep network.
# The first layer is an LSTM layer with 100 units followed by another LSTM layer with 50 units.
# Dropout is also applied after each LSTM layer to control overfitting.
# Final layer is a Dense output layer with single unit and sigmoid activation since this is a binary classification problem.
# build the network
nb_features = seq_array.shape[2]
nb_out = label_array.shape[1]
print("Num of GPU requested", G)
# Model Parallelism
if G <= 1:
print("[INFO] training with 1 GPU...")
model = lstm_model(sequence_length, nb_features, nb_out)
# otherwise, we are compiling using multiple GPUs
else:
print("[INFO] training with {} GPUs...".format(G))
# we'll store a copy of the model on *every* GPU and then combine
# the results from the gradient updates on the CPU
with tf.device("/cpu:0"):
# initialize the model
model = Sequential()
with tf.device('/gpu:0'):
model.add(LSTM(
input_shape=(sequence_length, nb_features),
units=100,
return_sequences=True))
# generate_predictions.py
# Anders Poirel
# 04-12-2019
import numpy as np
from lstm import lstm_model
CHECKPOINT_PATH = '../../output/checkpoint.ckpt'
DATA_PATH = '../../data/processed/'
model = lstm_model()
model.load_weights(CHECKPOINT_PATH)
X_test = np.load(DATA_PATH + 'X_test.npy')
y_pred = model.predict(X_test)
# predictions post-processing
row_maxes = y_pred.max(axis = 1).reshape(-1, 1)
y_pred[:] = np.where(y_pred == row_maxes, 1, 0)
y_pred = pd.DataFrame(y_pred, columns = ['1', '2', '3', '4', '5'])
y_pred = y_pred.idxmax(axis = 1).values.astype(np.int)
submission_df = pd.DataFrame({'Predictions' : y_pred})
submission_df.to_csv('predictions.csv',index = False)
dataset_val = keras.preprocessing.timeseries_dataset_from_array(
x_val,
y_val,
sequence_length=sequence_length,
sampling_rate=step,
batch_size=batch_size,
)
for batch in dataset_train.take(1):
inputs, targets = batch
print("Input shape:", inputs.numpy().shape)
print("Target shape:", targets.numpy().shape)
model = lstm_model(input_shape=(inputs.shape[1], inputs.shape[2]),
num_classes=1)
model.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate),
loss="mse")
model.summary()
keras.utils.plot_model(model, show_shapes=True)
path_checkpoint = "model_checkpoint.h5"
es_callback = keras.callbacks.EarlyStopping(monitor="val_loss",
min_delta=0,
patience=5)
modelckpt_callback = keras.callbacks.ModelCheckpoint(
monitor="val_loss",
filepath=path_checkpoint,