def train_model(stock_name, x_train, y_train, x_test, y_test, sc_close, unroll_length=50, retrain=False): batch_size = 32 epochs = 10 if retrain: model = lstm.lstm_model(x_train.shape[-1], output_dim=unroll_length, return_sequences=True) model.compile(loss='mean_squared_error', optimizer='adam') #start = time.time() model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, validation_split=0.05) model.save(os.getcwd() + "/data/" + stock_name + ".h5") else: if not os.path.isfile(os.getcwd() + "/data/" + stock_name + ".h5"): model = lstm.lstm_model(x_train.shape[-1], output_dim=unroll_length, return_sequences=True) model.compile(loss='mean_squared_error', optimizer='adam') #start = time.time() model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, validation_split=0.05) model.save(os.getcwd() + "/data/" + stock_name + ".h5") #print('training_time', time.time() - start) else: model = load_model(os.getcwd() + "/data/" + stock_name + ".h5") prediction = model.predict(x_test, batch_size=batch_size) prediction = sc_close.inverse_transform(prediction) #print(prediction) y_test = sc_close.inverse_transform(y_test) #vs.plot_lstm_prediction(y_test, prediction) return model, prediction
def __init__(self, fakeways=[], arp_for_unknowns=False, wide=False): # These are "fake gateways" -- we'll answer ARPs for them with MAC # of the switch they're connected to. self.fakeways = set(fakeways) # If True, we create "wide" matches. Otherwise, we create "narrow" # (exact) matches. self.wide = wide # If this is true and we see a packet for an unknown # host, we'll ARP for it. self.arp_for_unknowns = arp_for_unknowns # (dpid,IP) -> expire_time # We use this to keep from spamming ARPs self.outstanding_arps = {} # (dpid,IP) -> [(expire_time,buffer_id,in_port), ...] # These are buffers we've gotten at this datapath for this IP which # we can't deliver because we don't know where they go. self.lost_buffers = {} # For each switch, we map IP addresses to Entries self.arpTable = {} # This timer handles expiring stuff self._expire_timer = Timer(5, self._handle_expiration, recurring=True) core.listen_to_dependencies(self) ### edited ### print 'starting lstm switch' self.startTime = time.time() self.lstm = lstm.lstm_model()
def setUpClass(cls): """ initialize the lstm model """ cls.n_layers = 5 cls.batch_size = 20 cls.l = 10 # length of recursive chain np.random.seed(237) # the input and output dimensions of each layer cls.ns = np.array( [np.random.randint(1, 5) for i in range(cls.n_layers)]) cls.ms = np.concatenate((cls.ns[1:], [np.random.randint(1, 5)])) # the model cls.model = lstm_model(cls.ns, cls.ms, seed=102) # compile the network start = time.clock() cls.model.compile_network() cls.model_network_compile_time = time.clock() - start # save the weight matrices cls.Ws0 = cls.model.Ws.get_value() cls.Us0 = cls.model.Us.get_value() cls.bs0 = cls.model.bs.get_value() # tolerance for numerical optimization cls.tol = 1E-2 # learning rate cls.eta = 1. # a single layer model cls.nn = 3 cls.mm = 5 # one layer model for testing against by-hand gradients cls.one_layer_model = lstm_model(np.array([cls.nn]), np.array([cls.mm]), seed=1434)
def main(): clearLogFolder() load_russian_alphabet(ALPHABET_PATH, alphabet) if USE_SUPERTYPES: splitToSupertypes(supertypes) g = Graph() g = g.Read(data_in) get_all_seqs(g) if len(all_seq) > 0: print("Parsingsequences") X, y = processSequences(all_seq, TIMESTEPS) print("Merging sequences") Xm, ym = mergeSeqs(X, y) print("Create regressor") regressor = learn.SKCompat( learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS, optimizer="Adam"), model_dir=LOG_DIR)) # create a lstm instance and validation monitor validation_monitor = learn.monitors.ValidationMonitor( Xm['val'], ym['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000) print("fit regressor") regressor.fit(Xm['train'], ym['train'], monitors=[validation_monitor], batch_size=BATCH_SIZE, steps=TRAINING_STEPS) print("predicting") predicted = regressor.predict(Xm['test']) # rmse = np.sqrt(((predicted - ym['test']) ** 2).mean(axis=0)) score = mean_squared_error(predicted, ym['test']) hited = hitpoint(predicted, ym['test']) print("MSE: %f" % score) print("hitpoint:", hited)
# save dataset os.mkdir(data_path) pickle.dump( X, open( data_path + "/x_set" + str(lstm.IN_TIMESTEPS) + str(lstm.OUT_TIMESTEPS_RANGE[-1]) + ".pkl", "wb")) pickle.dump( Y, open( data_path + "/y_set" + str(lstm.IN_TIMESTEPS) + str(lstm.OUT_TIMESTEPS_RANGE[-1]) + ".pkl", "wb")) print("Save data successfully!") ## build the lstm model model_fn = lstm_model() config = tf.contrib.learn.RunConfig(log_step_count_steps=200, save_checkpoints_steps=VALIDATION_STEPS // 2) estimator = learn.Estimator(model_fn=model_fn, model_dir=LOG_DIR, config=config) regressor = learn.SKCompat(estimator) ## create a validation monitor validation_monitor = learn.monitors.ValidationMonitor( X['val'], Y['val'], every_n_steps=VALIDATION_STEPS) ## fit the train dataset # TRAINING_STEPS = 1
from data_processing import load_csvdata import xlrd import random import math import xlrd LOG_DIR = 'C:/Users/chak282/Downloads' TIMESTEPS = 3 RNN_LAYERS = [{'num_units': 5}] DENSE_LAYERS = None TRAINING_STEPS = 10000 PRINT_STEPS = TRAINING_STEPS / 10 BATCH_SIZE = 100 regressor = learn.SKCompat( learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS), model_dir=LOG_DIR)) workbook = xlrd.open_workbook('RNN_data.xlsx') sheet1 = workbook.sheet_by_name('RNN_data') data = np.zeros(shape=sheet1.nrows) for index1 in range(0, sheet1.nrows): data[index1] = sheet1.cell_value(index1, 1) X, y = load_csvdata(data, TIMESTEPS, seperate=False) # create a lstm instance and validation monitor validation_monitor = learn.monitors.ValidationMonitor( X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000) # print(X['train'])
from tensorflow.contrib import learn from sklearn.metrics import mean_squared_error from lstm import lstm_model from data_processing import generate_data tf.logging.set_verbosity(tf.logging.INFO) LOG_DIR = './ops_logs/sin' TIMESTEPS = 3 RNN_LAYERS = [{'num_units': 5}] DENSE_LAYERS = None TRAINING_STEPS = 10000 PRINT_STEPS = TRAINING_STEPS / 10 BATCH_SIZE = 100 regressor = learn.SKCompat( learn.Estimator( model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS), model_dir=LOG_DIR)) X, y = generate_data( np.sin, np.linspace(0, 100, 10000, dtype=np.float32), TIMESTEPS, seperate=False) # create a lstm instance and validation monitor validation_monitor = learn.monitors.ValidationMonitor(X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000) print(X['train']) print(y['train']) regressor.fit( X['train'], y['train'],
from sklearn.metrics import mean_squared_error from lstm import generate_data, lstm_model import logging logging.basicConfig(level=logging.DEBUG) LOG_DIR = '/tmp/m/ops_logs/6/ts' TIMESTEPS = 13 RNN_LAYERS = [{'num_units': 32},{'num_units': 64},{'num_units': 64}] DENSE_LAYERS = None TRAINING_STEPS = 40000 PRINT_STEPS = TRAINING_STEPS / 100 BATCH_SIZE = 20 regressor = learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS)) X, y = generate_data(np.sin, np.linspace(0, 100, 10000, dtype=np.float32), TIMESTEPS, seperate=False) ''' tushare data ''' import tushare as ts tdata = ts.get_hist_data('600848',ktype='d') ''' fdata = [] fdata.append([w for w in tdata.open]) fdata.append([w for w in tdata.close]) fdata.append([w for w in tdata.low]) fdata.append([w for w in tdata.high])
TIMESTEPS = 20 RNN_LAYERS = [{'num_units': 5}] DENSE_LAYERS = [10, 10] TRAINING_STEPS = 100000 BATCH_SIZE = 100 PRINT_STEPS = TRAINING_STEPS / 100 # We want to predict the Closing price. Not too much data close = pred.data.Close.ix[(len(pred.props) - 252 * 2):] # Split the data into test and train X, y = load_csvdata(close, TIMESTEPS, seperate=False) regressor = learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS, learning_rate=0.05, optimizer="Adagrad"), model_dir=LOG_DIR) # create a lstm instance and validation monitor validation_monitor = learn.monitors.ValidationMonitor( X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000) # Fit the data to the models regressor.fit(X['train'], y['train'], monitors=[validation_monitor], batch_size=BATCH_SIZE, steps=TRAINING_STEPS) # Make the prediction
realX = realX.astype('float32') realY = realY.astype('float32') # test data tX = range(0, 243) testX = [] for i in range(152, 182 + 61 - TIMESTEPS): testX.append([[x] for x in tX[i:i+TIMESTEPS]]) testX = np.array(testX) testX = testX.astype('float32') # model regressor = skflow.TensorFlowEstimator( model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS), n_classes=0, verbose=1, steps=TRAINING_STEPS, optimizer='Adagrad', learning_rate=0.03, batch_size=BATCH_SIZE) regressor.fit(realX, realY, logdir=LOG_DIR) predict = regressor.predict(testX) day = [(datetime.date(2015, 9, 1) + datetime.timedelta(x)).strftime("%Y%m%d") for x in range(60)] res = pd.DataFrame() res['artist_id'] = [a1 for x in range(60)] res['plays'] = predict[1:]
def DL_BuySell(price, days_future, pred,limit=0.0051, days_previous=504, train_split=0.9, periodM = 9, fastM = 12, slowM = 26, upperS = 80, lowerS = 20, daysS = 3, periodS = 14): """ This function takes the previous stock price and makes a LSTM deep learning network based on it to predict the future stock price. It is bound by a percentage change limit, stokementric oscillator and momentum considerations. :param price: pandas DataFrame, All technical details. generated from Predictors class :param days_future: pandas DateTime, The timestamps you want to predict :param limit: float, the minimum limit for which trades can occur :param days_previous: int, How many previous days should be simulated :param train_split: float, Training/Testing split between (0, 1) :param fastM: Period to calculate the fast mean (Exponential) :param periodM: MACD Smoothing number of days :param slowM: Period to calculate the slow mean (Exponential) :param upperS: Upper bound of the stokementric oscillator :param lowerS: Lower bound of the stokementric oscillator :param daysS: Period to calculate the mean (stokementric oscillator) :param periodS: Number of days to smooth the stokementric oscillator :return: pandas DataFrame containing Buy and Sell commands. """ LOG_DIR = '.opt_logs/lstm_stock' TIMESTEPS = 20 RNN_LAYERS = [{'num_units': 5}] DENSE_LAYERS = [10, 10] TRAINING_STEPS = 100000 BATCH_SIZE = 100 PRINT_STEPS = TRAINING_STEPS / 100 # Split into testing and training data. data = price.ix[price.index < days_future[0]] data = data.ix[-1*(days_previous):] nval = int(round(len(data) * (1 - train_split))) df_train, df_val, df_test = data.iloc[:nval], data.iloc[nval:], price.ix[days_future] labels = False train_x, val_x, test_x = (rnn_data(df_train, TIMESTEPS, labels=labels), rnn_data(df_val, TIMESTEPS, labels=labels), rnn_data(df_test, TIMESTEPS, labels=labels)) labels = True train_y, val_y, test_y = (rnn_data(df_train, TIMESTEPS, labels=labels), rnn_data(df_val, TIMESTEPS, labels=labels), rnn_data(df_test, TIMESTEPS, labels=labels)) X, y = dict(train=train_x, val=val_x, test=test_x), dict(train=train_y, val=val_y, test=test_y) # Train the model regressor = learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS, learning_rate=0.05, optimizer="Adagrad"), model_dir=LOG_DIR) # create a lstm instance and validation monitor validation_monitor = learn.monitors.ValidationMonitor(X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000) # Fit the data to the models regressor.fit(X['train'], y['train'], monitors=[validation_monitor], batch_size=BATCH_SIZE, steps=TRAINING_STEPS) # Make the prediction predicted = list(regressor.predict(X['test'])) # Can we make money? if (np.max(predicted)-np.min(predicted))/np.max(predicted) < limit: return index = price.ix[data.index[0]:days_future.index[-1]].index pred_val = np.concatenate((price.ix[data.index[0]:days_future.index[0]].values,predicted)) C = pd.DataFrame({"Close": pred_val}, index=index) stok_Sig = pred.stokO_signal(upper=upperS, lower=lowerS, days=daysS, period=periodS, C=C) MACD_Sig = pred.MACD_signal_line(period=periodM, hist=True, signal=True, fast=fastM, slow=slowM, C=C) # Where do all the signals overlap C["regime"] = (stok_Sig + MACD_Sig) C["regime"].ix[C["regime"] == 2] = 1 C["regime"].ix[C["regime"] == 1] = 0 C["regime"].ix[C["regime"] == 0] = 0 C["regime"].ix[C["regime"] == -1] = 0 C["regime"].ix[C["regime"] == -2] = -1 # We always have to end on a sell signal CC = C[C.regime != 0].copy() temp = CC['regime'].ix[-1] CC['regime'].ix[-1] = 0 CC['signal'] = np.sign(CC['regime'] - CC['regime'].shift(1)) CC['regime'].ix[-1] = temp # Make a dataframe containing the buy and sell signals for back testing. moves = pd.concat([ pd.DataFrame({"Price": CC.loc[CC["signal"] == 1, "Close"], "Regime": CC.loc[CC["signal"] == 1, 'regime'], "Signal": "Buy"}), pd.DataFrame({"Price": CC.loc[CC["signal"] == -1, "Close"], "Regime": CC.loc[CC["signal"] == -1, 'regime'], "Signal": "Sell"}) ]) moves.sort_index(inplace=True) if moves.ix[0].Regime == -1: moves = moves.ix[1:] return moves
from import_dataset import importer from lstm import lstm_model import matplotlib.pyplot as plt if __name__ == '__main__': df = importer() #plot plt.figure(figsize=(16,8)) plt.plot(df['Close'], label='Close Price history') lstm_model(df)
label_array.shape # Next, we build a deep network. # The first layer is an LSTM layer with 100 units followed by another LSTM layer with 50 units. # Dropout is also applied after each LSTM layer to control overfitting. # Final layer is a Dense output layer with single unit and sigmoid activation since this is a binary classification problem. # build the network nb_features = seq_array.shape[2] nb_out = label_array.shape[1] print("Num of GPU requested", G) # Model Parallelism if G <= 1: print("[INFO] training with 1 GPU...") model = lstm_model(sequence_length, nb_features, nb_out) # otherwise, we are compiling using multiple GPUs else: print("[INFO] training with {} GPUs...".format(G)) # we'll store a copy of the model on *every* GPU and then combine # the results from the gradient updates on the CPU with tf.device("/cpu:0"): # initialize the model model = Sequential() with tf.device('/gpu:0'): model.add(LSTM( input_shape=(sequence_length, nb_features), units=100, return_sequences=True))
# generate_predictions.py # Anders Poirel # 04-12-2019 import numpy as np from lstm import lstm_model CHECKPOINT_PATH = '../../output/checkpoint.ckpt' DATA_PATH = '../../data/processed/' model = lstm_model() model.load_weights(CHECKPOINT_PATH) X_test = np.load(DATA_PATH + 'X_test.npy') y_pred = model.predict(X_test) # predictions post-processing row_maxes = y_pred.max(axis = 1).reshape(-1, 1) y_pred[:] = np.where(y_pred == row_maxes, 1, 0) y_pred = pd.DataFrame(y_pred, columns = ['1', '2', '3', '4', '5']) y_pred = y_pred.idxmax(axis = 1).values.astype(np.int) submission_df = pd.DataFrame({'Predictions' : y_pred}) submission_df.to_csv('predictions.csv',index = False)
dataset_val = keras.preprocessing.timeseries_dataset_from_array( x_val, y_val, sequence_length=sequence_length, sampling_rate=step, batch_size=batch_size, ) for batch in dataset_train.take(1): inputs, targets = batch print("Input shape:", inputs.numpy().shape) print("Target shape:", targets.numpy().shape) model = lstm_model(input_shape=(inputs.shape[1], inputs.shape[2]), num_classes=1) model.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate), loss="mse") model.summary() keras.utils.plot_model(model, show_shapes=True) path_checkpoint = "model_checkpoint.h5" es_callback = keras.callbacks.EarlyStopping(monitor="val_loss", min_delta=0, patience=5) modelckpt_callback = keras.callbacks.ModelCheckpoint( monitor="val_loss", filepath=path_checkpoint,