def runNN1():
global ds
ds = dl.load_data(p.ticker, p.currency)
ds['VOL'] = ds['volume'] / ds['volume'].rolling(window=p.vol_period).mean()
ds['HH'] = ds['high'] / ds['high'].rolling(window=p.hh_period).max()
ds['LL'] = ds['low'] / ds['low'].rolling(window=p.ll_period).min()
ds['DR'] = ds['close'] / ds['close'].shift(1)
ds['MA'] = ds['close'] / ds['close'].rolling(window=p.sma_period).mean()
ds['MA2'] = ds['close'] / ds['close'].rolling(window=2 *
p.sma_period).mean()
ds['STD'] = ds['close'].rolling(p.std_period).std() / ds['close']
ds['RSI'] = talib.RSI(ds['close'].values, timeperiod=p.rsi_period)
ds['WR'] = talib.WILLR(ds['high'].values, ds['low'].values,
ds['close'].values, p.wil_period)
ds['DMA'] = ds.MA / ds.MA.shift(1)
ds['MAR'] = ds.MA / ds.MA2
if p.btc_data:
# p.reload must be True for this to work!
ds1 = dl.load_data('ETH', 'BTC')
ds = ds.join(ds1, on='time', rsuffix='_btc')
ds['RSI_BTC'] = talib.RSI(ds['close_btc'].values,
timeperiod=p.rsi_period)
ds['BTC/ETH'] = ds['close'] / ds['close_btc']
p.feature_list += ['RSI_BTC', 'BTC/ETH']
ds = ds.dropna()
td = train_test_nn(ds)
return td
def run_forecast(conf, seed=None):
global tdf
global df
if seed is not None: np.random.seed(seed)
init(conf)
dl.load_data() # Load Historical Price Data
# This needs to run before test dataset as it generates bin config
if p.train: df = dl.get_dataset() # Read Train data.
tdf = dl.get_dataset(test=True) # Read Test data
if p.train: train_model(df, tdf)
tdf = run_model(tdf, test=True)
if p.stats: show_result(tdf, "Test") # Test Result
print_forecast(tdf) # Print Forecast
if p.execute: t.execute_action()
def runNN2():
global ds
ds = dl.load_data(p.ticker, p.currency)
ds['DR'] = ds['close'] / ds['close'].shift(1)
ds['ADR'] = ds['DR'].rolling(window=14).mean()
calendar = dl.get_calendar(ds.date.min(), ds.date.max())
ds = pd.merge(calendar, ds, on='date', how='left')
ds = ds.dropna()
for col in calendar.columns:
if col == 'date':
continue
ds = dl.encode(ds, col, 359)
td = train_test_nn(ds)
return td
def runNN():
global td
global ds
ds = dl.load_data()
ds = add_features(ds)
# Separate input from output. Exclude last row
X = ds[p.feature_list][:-1]
# y = ds[['DR']].shift(-1)[:-1]
y = ds[['Price_Rise']].shift(-1)[:-1]
# Split Train and Test and scale
X_train, X_test, y_train, y_test = get_train_test(X, y)
K.clear_session() # Required to speed up model load
if p.train:
file = p.cfgdir + '/model.nn'
nn = train_model(X_train, X_test, y_train, y_test, file)
else:
file = p.model
nn = load_model(file)
# print('Loaded best model: '+file)
# Making prediction
y_pred_val = nn.predict(X_test)
# Generating Signals
td = gen_signal(ds, y_pred_val)
# Backtesting
td = run_backtest(td, file)
print(str(get_signal_str()))
def runLSTM():
global ds
global td
ds = dl.load_data()
ds = add_features(ds)
lag = 3
n_features = 1
X = pd.DataFrame()
for i in range(1, lag + 1):
X['RSI' + str(i)] = ds['RSI'].shift(i)
# X['MA'+str(i)] = ds['MA'].shift(i)
# X['VOL'+str(i)] = ds['VOL'].shift(i)
X = X.dropna()
y = ds['DR']
X_train, X_test, y_train, y_test = get_train_test(X, y)
X_train_t = X_train.reshape(X_train.shape[0], lag, n_features)
X_test_t = X_test.reshape(X_test.shape[0], lag, n_features)
file = p.model
if p.train:
file = p.cfgdir + '/model.nn'
nn = Sequential()
nn.add(
LSTM(p.units,
input_shape=(X_train_t.shape[1], X_train_t.shape[2]),
return_sequences=True))
nn.add(Dropout(0.2))
nn.add(LSTM(p.units, return_sequences=False))
nn.add(Dense(1))
optimizer = RMSprop(lr=0.005, clipvalue=1.)
# optimizer = 'adam'
nn.compile(loss=p.loss, optimizer=optimizer)
cp = ModelCheckpoint(file,
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='min')
h = nn.fit(X_train_t,
y_train,
batch_size=10,
epochs=p.epochs,
verbose=0,
callbacks=[cp],
validation_data=(X_test_t, y_test))
plot_fit_history(h)
# Load Best Model
nn = load_model(file)
y_pred = nn.predict(X_test_t)
td = gen_signal(ds, y_pred)
# Backtesting
td = run_backtest(td, file)
print(str(get_signal_str()))
def runNN1():
global td
global ds
ds = dl.load_data()
ds['VOL'] = ds['volume'] / ds['volume'].rolling(window=p.vol_period).mean()
ds['HH'] = ds['high'] / ds['high'].rolling(window=p.hh_period).max()
ds['LL'] = ds['low'] / ds['low'].rolling(window=p.ll_period).min()
ds['DR'] = ds['close'] / ds['close'].shift(1)
ds['MA'] = ds['close'] / ds['close'].rolling(window=p.sma_period).mean()
ds['MA2'] = ds['close'] / ds['close'].rolling(window=2 *
p.sma_period).mean()
ds['STD'] = ds['close'].rolling(p.std_period).std() / ds['close']
ds['RSI'] = talib.RSI(ds['close'].values, timeperiod=p.rsi_period)
ds['WR'] = talib.WILLR(ds['high'].values, ds['low'].values,
ds['close'].values, p.wil_period)
ds['DMA'] = ds.MA / ds.MA.shift(1)
ds['MAR'] = ds.MA / ds.MA2
ds['ADX'] = talib.ADX(ds['high'].values,
ds['low'].values,
ds['close'].values,
timeperiod=p.adx_period)
ds['Price_Rise'] = np.where(ds['DR'] > 1, 1, 0)
if p.btc_data:
p.currency = 'BTC'
p.kraken_pair = 'XETHXXBT'
ds1 = dl.load_data()
ds = ds.join(ds1, rsuffix='_btc')
ds['RSI_BTC'] = talib.RSI(ds['close_btc'].values,
timeperiod=p.rsi_period)
p.feature_list += ['RSI_BTC']
ds = ds.dropna()
# Separate input from output. Exclude last row
X = ds[p.feature_list][:-1]
y = ds[['DR']].shift(-1)[:-1]
# Split Train and Test and scale
train_split = int(len(X) * p.train_pct)
test_split = p.test_bars if p.test_bars > 0 else int(len(X) * p.test_pct)
X_train, X_test, y_train, y_test = X[:train_split], X[
-test_split:], y[:train_split], y[-test_split:]
# Feature Scaling
# from sklearn.preprocessing import QuantileTransformer, MinMaxScaler
scaler = p.cfgdir + '/sc.dmp'
scaler1 = p.cfgdir + '/sc1.dmp'
if p.train:
# sc = QuantileTransformer(10)
# sc = MinMaxScaler()
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
dump(sc, scaler)
sc1 = MinMaxScaler()
y_train = sc1.fit_transform(y_train)
y_test = sc1.transform(y_test)
dump(sc1, scaler1)
else:
sc = load(scaler)
X_train = sc.transform(X_train)
X_test = sc.transform(X_test)
sc1 = load(scaler1)
y_train = sc1.transform(y_train)
y_test = sc1.transform(y_test)
K.clear_session() # Required to speed up model load
if p.train:
# Custom Loss Function
# def stock_loss(t, p):
# loss = K.switch(K.less((t-1)*(p-1), 0), K.abs(t-p), 0.1*K.abs(t-p))
# return K.mean(loss, axis=-1)
# p.loss = stock_loss
file = p.cfgdir + '/model.nn'
print('*** Training model with ' + str(p.units) +
' units per layer ***')
nn = Sequential()
nn.add(
Dense(units=p.units,
kernel_initializer='uniform',
activation='relu',
input_dim=X_train.shape[1]))
nn.add(
Dense(units=p.units,
kernel_initializer='uniform',
activation='relu'))
nn.add(
Dense(units=1, kernel_initializer='uniform', activation='linear'))
cp = ModelCheckpoint(file,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
nn.compile(optimizer='adam', loss=p.loss, metrics=['accuracy'])
history = nn.fit(
X_train,
y_train,
batch_size=len(X_train) if p.batch_size == 0 else p.batch_size,
epochs=p.epochs,
callbacks=[cp],
validation_data=(X_test, y_test),
verbose=0)
# Plot model history
plot_fit_history(history)
# Load Best Model
# nn = load_model(file, custom_objects={'stock_loss': stock_loss})
nn = load_model(file)
else:
file = p.model
nn = load_model(file)
# Making prediction
y_pred_val = nn.predict(X_test)
y_pred_val = sc1.inverse_transform(y_pred_val)
# Generating Signals
td = gen_signal(ds, y_pred_val)
# Backtesting
td = run_backtest(td, file)
print(str(get_signal_str()))
# pprint(a)
# pprint(meta_path)
# pprint(smpl)
if bid is not None:
sample_path = smpl['value']
meta_path = meta_path.bags[bid][1]
return sample
# --- load data
# data_raw = json.load( open(f"../data/stats_raw_full.json", "r") )
data_raw = json.load(open(f"../data/{args.dataset}.json", "r"))
data, meta, meta_full = datalib.load_data(config.DATA_FILE, config.META_FILE)
config.init_dataset(meta_full)
label_id = meta_full['label']
label_type = meta_full['description'][label_id]['type']
print("config =", config)
print(
f"Using dataset {meta_full['name']} with {meta_full['samples']} samples and {meta_full['classes']} classes."
)
data_trn, data_val, data_tst, shuffle_idx = datalib.split(
data, config.DATASEED)
net = Net(meta)
env = SeqEnv(data_tst, meta)
tsret['Lag%s' % str(i+1)] = \
tslag['Lag%s' % str(i+1)].pct_change() * 100.0
# Create the "Direction" column (+1 or -1) indicating an up/down day
tsret['Direction'] = np.sign(tsret['Today'])
tsret = tsret.dropna()
return tsret
if __name__ == '__main__':
p.load_config('ETHUSDNN')
p.datasource = 'cc'
p.signal_threshold = 0
ts = dl.load_data().set_index('date')
ts['date'] = ts.index
ds = create_lagged_series(ts, lags=5)
# Use the prior two days of returns as predictor
# values, with direction as the response
X = ds[['Lag1', 'Lag2']]
y = ds['Direction']
# The test data is split into two parts: Before and after 1st Jan 2017.
start_test = dt(2019, 1, 1)
# Create training and test sets
X_train = X[X.index < start_test]
X_test = X[X.index >= start_test]
