def get_heiken_cdv_obs_(self, ind, df):
df = df.loc[:ind]
df = df.tail(self.nPoints).copy()
df = FeatGen_HeikenAshi().transform(df, verbose=False)
df = self.fg_cdv.transform(df, self.ema_period_volume)
df = df.iloc[self.ema_period_volume - 1:]
df = df.tail(self.window_size)
del df["tick_volume"]
scaler = MinMaxScaler(feature_range=(-1, 1))
for feat in ["open", "high", "low", "close"]:
tmp = df[feat].values.reshape((-1, 1))
scaler.partial_fit(tmp)
for feat in ["open", "high", "low", "close"]:
tmp = df[feat].values.reshape((-1, 1))
tmp = scaler.transform(tmp)
tmp = tmp.reshape((-1, ))
df[feat] = tmp
for feat in ["cdv"]:
tmp = df[feat].values.reshape((-1, 1))
tmp = MinMaxScaler(feature_range=(-1, 1)).fit_transform(tmp)
tmp = tmp.reshape((-1, ))
df[feat] = tmp
#PlotRender().plot_price_cdv(df)
obs = df.values
obs = obs.reshape((1, ) + obs.shape + (1, ))
return obs
def __init__(self, use_heiken_ashi):
self.needful_cols = ["open", "high", "low", "close", "tick_volume"]
if use_heiken_ashi:
self.gen_hk = FeatGen_HeikenAshi()
else:
self.gen_hk = None
self.gen_cdv = FeatGen_CDV()
pass
from classes.delta_bender.SymbolDataManager import SymbolDataManager
from classes.delta_bender.FeatGen_HeikenAshi import FeatGen_HeikenAshi
from classes.delta_bender.FeatGen_CDV import FeatGen_CDV
from classes.delta_bender.FeatGen_CandleRelations import FeatGen_CandleRelations
from classes.delta_bender.FeatGen_ClusterLabels import FeatGen_ClusterLabels
from classes.delta_bender.PlotRender import PlotRender
import numpy as np
symbol = "EURUSD_i"
timeframe = "M15"
dataManager = SymbolDataManager("../data/raw/")
df = dataManager.getData(symbol, timeframe)
df = df[["open", "high", "low", "close", "tick_volume"]]
gen_hk = FeatGen_HeikenAshi()
gen_cr = FeatGen_CandleRelations()
gen_cluster = FeatGen_ClusterLabels()
#df = df.tail(5000)
df = gen_hk.transform(df)
df_for_clusterizer = gen_cr.fit_transform(df)
gen_cluster.fit(df_for_clusterizer, verbose=True)
#df = df.tail(1000)
df = gen_cr.transform(df)
df = gen_cluster.transform(df)
df = FeatGen_CDV().transform(df, period=32)
"""y_cluster = df["cluster_label"].valuesdf = df.drop( ["open", "high", "low", "close", "tick_volume", "cluster_label", "cdv"], axis=1 )
vals = df.values
uniq_y = np.unique( y_cluster )
x_tsne = TSNE(perplexity=60, n_iter=1000, n_jobs=8, verbose=True).fit_transform(vals)
for uy in uniq_y:
import os
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
symbol = "EURUSD_i"
timeframe = "M5"
target_columns = ["open", "high", "low", "close", "tick_volume"]
window_size = 64
ema_period_volume = 14
ema_period_price = 3
dataManager = SymbolDataManager("../data/raw/")
df = dataManager.getData(symbol, timeframe)
df = df[target_columns]
df = df.tail(1000)
df = FeatGen_HeikenAshi().transform(df)
df = FeatGen_CDV().transform(df, ema_period_volume)
df = df.iloc[ema_period_volume - 1:]
#df = FeatGen_PriceEMA().transform(df, ema_period_price, verbose=False)
#df = df.iloc[ema_period_price - 1:]
#df = FeatGen_CDV().transform(df, ema_period_volume)
#df = df.iloc[ema_period_volume - 1:]
del df["tick_volume"]
df_vals = df.values
print(df.shape)
def proc_symbol_timeframe_pair( symbol, timeframe ):
feat_window_size = 64
ema_period_volume = 14
ema_period_price = 3
close_delta_window = 6
feat_gen = FeatGen_ScaledWindow(["open", "high", "low", "close", "cdv"], nPoints=feat_window_size, nDiffs=0, flatStack=False)
dataManager = SymbolDataManager("../data/raw/")
print(symbol + ": " + timeframe)
df = dataManager.getData(symbol, timeframe)
df = df[target_columns]
df = FeatGen_HeikenAshi().transform(df)
df = FeatGen_CDV().transform(df, ema_period_volume)
df = df.iloc[ema_period_volume - 1:]
#df = FeatGen_PriceEMA().transform(df, ema_period_price, verbose=False)
#df = df.iloc[ema_period_price - 1:]
#df = FeatGen_CDV().transform(df, ema_period_volume)
#df = df.iloc[ema_period_volume - 1:]
del df["tick_volume"]
###############################
df_vals = df.values
change_deltas = []
for i in range(1, len(df_vals) - close_delta_window):
prev_candle = df_vals[i - 1]
mean_delta = []
for j in range(close_delta_window):
current_candle = df_vals[i + j]
mean_delta.append( abs(current_candle[3] - prev_candle[3]) )
mean_delta = np.average( mean_delta )
change_deltas.append(mean_delta)
# change_treshold = np.percentile(change_deltas, change_percentile)
change_deltas = pd.Series(change_deltas)
quantiles = pd.qcut(change_deltas, 20, retbins=True)[1]
################################
x = []
y = []
for i in tqdm(range(df.index.values[0] + 1, df.index.values[-1] - feat_window_size - close_delta_window - 1), desc="generating dataset",
colour="green"):
current_candle = df.loc[i + feat_window_size]
mean_delta = []
for j in range(close_delta_window):
next_candle = df.loc[i + j + feat_window_size + 1]
mean_delta.append(next_candle[3] - current_candle[3])
mean_delta = np.average(mean_delta)
current_delta_sign = np.sign(mean_delta)
abs_delta = abs(mean_delta)
# current_delta = abs(((current_candle[3] - prev_candle[3]) + (current_candle[0] - prev_candle[0])) / 2.0)
min_delta_treshold = quantiles[-7] #70%
max_delta_treshold = quantiles[-2] #95%
if abs_delta < min_delta_treshold:
y_i = 0
elif abs_delta > min_delta_treshold and abs_delta < max_delta_treshold and current_delta_sign == -1:
y_i = -1
elif abs_delta > min_delta_treshold and abs_delta < max_delta_treshold and current_delta_sign == 1:
y_i = 1
else:
y_i = None
if y_i in [-1, 1]:
y.append(y_i)
ind = i + feat_window_size
######
# DEBUG
#ind += 1
######
x_i = feat_gen.get_window(ind, df)
x.append(x_i)
x = np.vstack(x)
y = np.array(y)
print(x.shape)
save(x, "../data/model_datasets/x_{}_{}.pkl".format(symbol, timeframe))
save(y, "../data/model_datasets/y_{}_{}.pkl".format(symbol, timeframe))
from classes.delta_bender.SymbolDataManager import SymbolDataManager
from classes.delta_bender.FeatGen_HeikenAshi import FeatGen_HeikenAshi
from classes.delta_bender.FeatGen_CDV import FeatGen_CDV
from classes.delta_bender import PlotRender
symbol = "EURUSD_i"
timeframe = "M15"
dataManager = SymbolDataManager("../data/raw/")
df = dataManager.getData(symbol, timeframe)
print(df)
df = df.tail(1200)
df = FeatGen_HeikenAshi().transform(df, verbose=True)
#PlotRender().plot_candles(df)
df = FeatGen_CDV().transform(df, period=32, verbose=True)
#PlotRender().plot_cdv(df)
PlotRender().plot_price_cdv(df)
print("done")
class FeatGenMeta():
def __init__(self, use_heiken_ashi):
self.needful_cols = ["open", "high", "low", "close", "tick_volume"]
if use_heiken_ashi:
self.gen_hk = FeatGen_HeikenAshi()
else:
self.gen_hk = None
self.gen_cdv = FeatGen_CDV()
pass
def transform(self, x, ema_period, n_price_feats=None, m_cdv_feats=None):
if isinstance(x, np.ndarray):
x = pd.DataFrame(x, columns=self.needful_cols)
elif isinstance(x, pd.DataFrame):
x.reset_index(drop=True, inplace=True)
if self.gen_hk is not None:
x = self.gen_hk.transform(x)
x = self.gen_cdv.transform(x, period=ema_period, verbose=False)
x = x[ema_period - 1:]
x.reset_index(drop=True, inplace=True)
cdv_vals = x["cdv"].values
cdv_change_ind = []
for i in range(1, len(cdv_vals)):
cdv_change_ind.append(cdv_vals[i] - cdv_vals[i - 1])
"""if cdv_vals[i] > cdv_vals[i-1]:
cdv_change_ind.append( 1 )
elif cdv_vals[i] < cdv_vals[i-1]:
cdv_change_ind.append( -1 )
else:
cdv_change_ind.append( 0 )"""
cdv_change_ind = np.array(cdv_change_ind).reshape((-1, 1))
cdv_change_ind = MinMaxScaler(
feature_range=(-1, 1)).fit_transform(cdv_change_ind)
cdv_change_ind = np.array(cdv_change_ind).reshape((-1, ))
x = x[1:]
x.reset_index(drop=True, inplace=True)
x["cdv_change_ind"] = cdv_change_ind
cdv_vals = np.array(cdv_vals).reshape((-1, 1))
cdv_vals = MinMaxScaler(feature_range=(-1, 1)).fit_transform(cdv_vals)
cdv_vals = np.array(cdv_vals).reshape((-1, ))
if m_cdv_feats is not None:
cdv_vals = cdv_vals[-m_cdv_feats:]
cdv_change_feats = x["cdv_change_ind"].values[-m_cdv_feats:]
else:
cdv_change_feats = x["cdv_change_ind"].values
price_feats = self.get_price_feats_(x, n_price_feats)
feats = np.hstack([price_feats, cdv_vals, cdv_change_feats])
return feats
def get_price_feats_(self, df, n):
df = df.copy()
if n is not None:
df = df.tail(n)
a = np.array([i for i in range(n)]).reshape((-1, 1))
x = np.vstack([a, a, a, a])
y = np.hstack([
df["open"].values, df["high"].values, df["low"].values,
df["close"].values
])
y = np.array(y).reshape((-1, 1))
y = MinMaxScaler(feature_range=(-1, 1)).fit_transform(y)
y = np.array(y).reshape((-1, ))
model = Ridge(random_state=45).fit(x, y)
y_pred = model.predict(a)
scaled_prices = []
for i in range(4):
price_property = y[i * n:(i + 1) * n]
scaled_prices.append(price_property)
scaled_prices = np.array(scaled_prices)
price_deltas = scaled_prices - y_pred
means = np.mean(price_deltas, axis=0)
stds = np.std(price_deltas, axis=0)
min_deltas = np.min(price_deltas, axis=0)
max_deltas = np.max(price_deltas, axis=0)
std_global = np.std(price_deltas)
min_deltas_global = np.min(price_deltas)
max_deltas_global = np.max(price_deltas)
angle = self.get_angle_(0.0, y_pred[0], 1.0, y_pred[1])
price_feats = np.hstack([
y, means, stds, min_deltas, max_deltas,
[std_global, min_deltas_global, max_deltas_global, angle]
])
return price_feats
def get_angle_(self, x_1, y_1, x_2, y_2):
x = x_2 - x_1
y = y_2 - y_1
cos_x = x / np.sqrt(np.square(x) + np.square(y))
sin_x = np.sqrt(1 - np.square(cos_x))
return sin_x