def runNN1(): global ds ds = dl.load_data(p.ticker, p.currency) ds['VOL'] = ds['volume'] / ds['volume'].rolling(window=p.vol_period).mean() ds['HH'] = ds['high'] / ds['high'].rolling(window=p.hh_period).max() ds['LL'] = ds['low'] / ds['low'].rolling(window=p.ll_period).min() ds['DR'] = ds['close'] / ds['close'].shift(1) ds['MA'] = ds['close'] / ds['close'].rolling(window=p.sma_period).mean() ds['MA2'] = ds['close'] / ds['close'].rolling(window=2 * p.sma_period).mean() ds['STD'] = ds['close'].rolling(p.std_period).std() / ds['close'] ds['RSI'] = talib.RSI(ds['close'].values, timeperiod=p.rsi_period) ds['WR'] = talib.WILLR(ds['high'].values, ds['low'].values, ds['close'].values, p.wil_period) ds['DMA'] = ds.MA / ds.MA.shift(1) ds['MAR'] = ds.MA / ds.MA2 if p.btc_data: # p.reload must be True for this to work! ds1 = dl.load_data('ETH', 'BTC') ds = ds.join(ds1, on='time', rsuffix='_btc') ds['RSI_BTC'] = talib.RSI(ds['close_btc'].values, timeperiod=p.rsi_period) ds['BTC/ETH'] = ds['close'] / ds['close_btc'] p.feature_list += ['RSI_BTC', 'BTC/ETH'] ds = ds.dropna() td = train_test_nn(ds) return td
def run_forecast(conf, seed=None): global tdf global df if seed is not None: np.random.seed(seed) init(conf) dl.load_data() # Load Historical Price Data # This needs to run before test dataset as it generates bin config if p.train: df = dl.get_dataset() # Read Train data. tdf = dl.get_dataset(test=True) # Read Test data if p.train: train_model(df, tdf) tdf = run_model(tdf, test=True) if p.stats: show_result(tdf, "Test") # Test Result print_forecast(tdf) # Print Forecast if p.execute: t.execute_action()
def runNN2(): global ds ds = dl.load_data(p.ticker, p.currency) ds['DR'] = ds['close'] / ds['close'].shift(1) ds['ADR'] = ds['DR'].rolling(window=14).mean() calendar = dl.get_calendar(ds.date.min(), ds.date.max()) ds = pd.merge(calendar, ds, on='date', how='left') ds = ds.dropna() for col in calendar.columns: if col == 'date': continue ds = dl.encode(ds, col, 359) td = train_test_nn(ds) return td
def runNN(): global td global ds ds = dl.load_data() ds = add_features(ds) # Separate input from output. Exclude last row X = ds[p.feature_list][:-1] # y = ds[['DR']].shift(-1)[:-1] y = ds[['Price_Rise']].shift(-1)[:-1] # Split Train and Test and scale X_train, X_test, y_train, y_test = get_train_test(X, y) K.clear_session() # Required to speed up model load if p.train: file = p.cfgdir + '/model.nn' nn = train_model(X_train, X_test, y_train, y_test, file) else: file = p.model nn = load_model(file) # print('Loaded best model: '+file) # Making prediction y_pred_val = nn.predict(X_test) # Generating Signals td = gen_signal(ds, y_pred_val) # Backtesting td = run_backtest(td, file) print(str(get_signal_str()))
def runLSTM(): global ds global td ds = dl.load_data() ds = add_features(ds) lag = 3 n_features = 1 X = pd.DataFrame() for i in range(1, lag + 1): X['RSI' + str(i)] = ds['RSI'].shift(i) # X['MA'+str(i)] = ds['MA'].shift(i) # X['VOL'+str(i)] = ds['VOL'].shift(i) X = X.dropna() y = ds['DR'] X_train, X_test, y_train, y_test = get_train_test(X, y) X_train_t = X_train.reshape(X_train.shape[0], lag, n_features) X_test_t = X_test.reshape(X_test.shape[0], lag, n_features) file = p.model if p.train: file = p.cfgdir + '/model.nn' nn = Sequential() nn.add( LSTM(p.units, input_shape=(X_train_t.shape[1], X_train_t.shape[2]), return_sequences=True)) nn.add(Dropout(0.2)) nn.add(LSTM(p.units, return_sequences=False)) nn.add(Dense(1)) optimizer = RMSprop(lr=0.005, clipvalue=1.) # optimizer = 'adam' nn.compile(loss=p.loss, optimizer=optimizer) cp = ModelCheckpoint(file, monitor='val_loss', verbose=0, save_best_only=True, mode='min') h = nn.fit(X_train_t, y_train, batch_size=10, epochs=p.epochs, verbose=0, callbacks=[cp], validation_data=(X_test_t, y_test)) plot_fit_history(h) # Load Best Model nn = load_model(file) y_pred = nn.predict(X_test_t) td = gen_signal(ds, y_pred) # Backtesting td = run_backtest(td, file) print(str(get_signal_str()))
def runNN1(): global td global ds ds = dl.load_data() ds['VOL'] = ds['volume'] / ds['volume'].rolling(window=p.vol_period).mean() ds['HH'] = ds['high'] / ds['high'].rolling(window=p.hh_period).max() ds['LL'] = ds['low'] / ds['low'].rolling(window=p.ll_period).min() ds['DR'] = ds['close'] / ds['close'].shift(1) ds['MA'] = ds['close'] / ds['close'].rolling(window=p.sma_period).mean() ds['MA2'] = ds['close'] / ds['close'].rolling(window=2 * p.sma_period).mean() ds['STD'] = ds['close'].rolling(p.std_period).std() / ds['close'] ds['RSI'] = talib.RSI(ds['close'].values, timeperiod=p.rsi_period) ds['WR'] = talib.WILLR(ds['high'].values, ds['low'].values, ds['close'].values, p.wil_period) ds['DMA'] = ds.MA / ds.MA.shift(1) ds['MAR'] = ds.MA / ds.MA2 ds['ADX'] = talib.ADX(ds['high'].values, ds['low'].values, ds['close'].values, timeperiod=p.adx_period) ds['Price_Rise'] = np.where(ds['DR'] > 1, 1, 0) if p.btc_data: p.currency = 'BTC' p.kraken_pair = 'XETHXXBT' ds1 = dl.load_data() ds = ds.join(ds1, rsuffix='_btc') ds['RSI_BTC'] = talib.RSI(ds['close_btc'].values, timeperiod=p.rsi_period) p.feature_list += ['RSI_BTC'] ds = ds.dropna() # Separate input from output. Exclude last row X = ds[p.feature_list][:-1] y = ds[['DR']].shift(-1)[:-1] # Split Train and Test and scale train_split = int(len(X) * p.train_pct) test_split = p.test_bars if p.test_bars > 0 else int(len(X) * p.test_pct) X_train, X_test, y_train, y_test = X[:train_split], X[ -test_split:], y[:train_split], y[-test_split:] # Feature Scaling # from sklearn.preprocessing import QuantileTransformer, MinMaxScaler scaler = p.cfgdir + '/sc.dmp' scaler1 = p.cfgdir + '/sc1.dmp' if p.train: # sc = QuantileTransformer(10) # sc = MinMaxScaler() sc = StandardScaler() X_train = sc.fit_transform(X_train) X_test = sc.transform(X_test) dump(sc, scaler) sc1 = MinMaxScaler() y_train = sc1.fit_transform(y_train) y_test = sc1.transform(y_test) dump(sc1, scaler1) else: sc = load(scaler) X_train = sc.transform(X_train) X_test = sc.transform(X_test) sc1 = load(scaler1) y_train = sc1.transform(y_train) y_test = sc1.transform(y_test) K.clear_session() # Required to speed up model load if p.train: # Custom Loss Function # def stock_loss(t, p): # loss = K.switch(K.less((t-1)*(p-1), 0), K.abs(t-p), 0.1*K.abs(t-p)) # return K.mean(loss, axis=-1) # p.loss = stock_loss file = p.cfgdir + '/model.nn' print('*** Training model with ' + str(p.units) + ' units per layer ***') nn = Sequential() nn.add( Dense(units=p.units, kernel_initializer='uniform', activation='relu', input_dim=X_train.shape[1])) nn.add( Dense(units=p.units, kernel_initializer='uniform', activation='relu')) nn.add( Dense(units=1, kernel_initializer='uniform', activation='linear')) cp = ModelCheckpoint(file, monitor='val_loss', verbose=1, save_best_only=True, mode='min') nn.compile(optimizer='adam', loss=p.loss, metrics=['accuracy']) history = nn.fit( X_train, y_train, batch_size=len(X_train) if p.batch_size == 0 else p.batch_size, epochs=p.epochs, callbacks=[cp], validation_data=(X_test, y_test), verbose=0) # Plot model history plot_fit_history(history) # Load Best Model # nn = load_model(file, custom_objects={'stock_loss': stock_loss}) nn = load_model(file) else: file = p.model nn = load_model(file) # Making prediction y_pred_val = nn.predict(X_test) y_pred_val = sc1.inverse_transform(y_pred_val) # Generating Signals td = gen_signal(ds, y_pred_val) # Backtesting td = run_backtest(td, file) print(str(get_signal_str()))
# pprint(a) # pprint(meta_path) # pprint(smpl) if bid is not None: sample_path = smpl['value'] meta_path = meta_path.bags[bid][1] return sample # --- load data # data_raw = json.load( open(f"../data/stats_raw_full.json", "r") ) data_raw = json.load(open(f"../data/{args.dataset}.json", "r")) data, meta, meta_full = datalib.load_data(config.DATA_FILE, config.META_FILE) config.init_dataset(meta_full) label_id = meta_full['label'] label_type = meta_full['description'][label_id]['type'] print("config =", config) print( f"Using dataset {meta_full['name']} with {meta_full['samples']} samples and {meta_full['classes']} classes." ) data_trn, data_val, data_tst, shuffle_idx = datalib.split( data, config.DATASEED) net = Net(meta) env = SeqEnv(data_tst, meta)
tsret['Lag%s' % str(i+1)] = \ tslag['Lag%s' % str(i+1)].pct_change() * 100.0 # Create the "Direction" column (+1 or -1) indicating an up/down day tsret['Direction'] = np.sign(tsret['Today']) tsret = tsret.dropna() return tsret if __name__ == '__main__': p.load_config('ETHUSDNN') p.datasource = 'cc' p.signal_threshold = 0 ts = dl.load_data().set_index('date') ts['date'] = ts.index ds = create_lagged_series(ts, lags=5) # Use the prior two days of returns as predictor # values, with direction as the response X = ds[['Lag1', 'Lag2']] y = ds['Direction'] # The test data is split into two parts: Before and after 1st Jan 2017. start_test = dt(2019, 1, 1) # Create training and test sets X_train = X[X.index < start_test] X_test = X[X.index >= start_test]