def __init__(self, use_heiken_ashi):
self.needful_cols = ["open", "high", "low", "close", "tick_volume"]
if use_heiken_ashi:
self.gen_hk = FeatGen_HeikenAshi()
else:
self.gen_hk = None
self.gen_cdv = FeatGen_CDV()
pass
def __init__(self, agentName, model):
self.name = agentName
self.batch_size = 200
self.epsilon = 0
self.memory = []
self.nPoints = 200
self.ema_period_volume = 14
self.window_size = 64
self.fg_sw = FeatGen_ScaledWindow(["open", "high", "low", "close", "cdv"], nPoints=self.window_size, nDiffs=0,
flatStack=False)
self.fg_cdv = FeatGen_CDV()
self.fg_pe = FeatGen_PriceEMA()
self.model = model
def __init__(self,
featureList,
nPoints=256,
window_size=64,
ema_period_volume=14,
ema_period_price=3):
self.nPoints = nPoints
self.featureList = featureList
self.window_size = window_size
self.ema_period_volume = ema_period_volume
self.ema_period_price = ema_period_price
self.fg_sw = FeatGen_ScaledWindow(
["open", "high", "low", "close", "cdv"],
nPoints=self.window_size,
nDiffs=0,
flatStack=False)
self.fg_cdv = FeatGen_CDV()
self.fg_pe = FeatGen_PriceEMA()
self.featureShape = (self.window_size, 5)
pass
def get_action(self, df):
df = df.copy()
df = df[["open", "close", "low", "high", "tick_volume"]]
window_size = 64
ema_period_volume = 14
ema_period_price = 3
df = FeatGen_CDV().transform(df, ema_period_volume)
df = df.iloc[ema_period_volume - 1:]
df = FeatGen_PriceEMA().transform(df, ema_period_price, verbose=False)
df = df.iloc[ema_period_price - 1:]
del df["tick_volume"]
feat_gen = FeatGen_ScaledWindow(
["open", "high", "low", "close", "cdv"],
nPoints=window_size,
nDiffs=0,
flatStack=False)
ind = df.index.values[-1]
x_i = feat_gen.get_window(ind, df)
y_pred = self.model.predict(x_i)[0]
bear_proba = y_pred[0]
bull_proba = y_pred[1]
proba_treshold = 0.8
"""if bull_proba > bear_proba and bull_proba > proba_treshold:
action = 0
elif bear_proba > bear_proba and bear_proba > proba_treshold:
action = 1
else:
action = 0"""
action = 0
#if bull_proba > proba_treshold:
# action = 0 # reverse for fun
#else:
# action = 1
return action
dataManager = SymbolDataManager("../data/raw/")
df = dataManager.getData(symbol, timeframe)
df = df[["open", "high", "low", "close", "tick_volume"]]
gen_hk = FeatGen_HeikenAshi()
gen_cr = FeatGen_CandleRelations()
gen_cluster = FeatGen_ClusterLabels()
#df = df.tail(5000)
df = gen_hk.transform(df)
df_for_clusterizer = gen_cr.fit_transform(df)
gen_cluster.fit(df_for_clusterizer, verbose=True)
#df = df.tail(1000)
df = gen_cr.transform(df)
df = gen_cluster.transform(df)
df = FeatGen_CDV().transform(df, period=32)
"""y_cluster = df["cluster_label"].valuesdf = df.drop( ["open", "high", "low", "close", "tick_volume", "cluster_label", "cdv"], axis=1 )
vals = df.values
uniq_y = np.unique( y_cluster )
x_tsne = TSNE(perplexity=60, n_iter=1000, n_jobs=8, verbose=True).fit_transform(vals)
for uy in uniq_y:
plt.scatter( x_tsne[ y_cluster == uy, 0], x_tsne[ y_cluster == uy, 1], s=2 )
plt.show()"""
y_cluster = df["cluster_label"].values
uniq_y = np.unique(y_cluster)
print(len(uniq_y))
for uy in uniq_y:
cluster_df = df[df["cluster_label"] == uy]
cluster_df = cluster_df.iloc[:2000]
class DBAgentBuyer_HACDV:
def __init__(self, agentName, model):
self.name = agentName
self.batch_size = 200
self.epsilon = 0
self.memory = []
self.nPoints = 200
self.ema_period_volume = 14
self.window_size = 64
self.fg_sw = FeatGen_ScaledWindow(
["open", "high", "low", "close", "cdv"],
nPoints=self.window_size,
nDiffs=0,
flatStack=False)
self.fg_cdv = FeatGen_CDV()
self.fg_pe = FeatGen_PriceEMA()
self.model = model
def get_heiken_cdv_obs_(self, df):
df = df.tail(self.nPoints).copy()
df = FeatGen_HeikenAshi().transform(df, verbose=False)
df = self.fg_cdv.transform(df, self.ema_period_volume)
df = df.iloc[self.ema_period_volume - 1:]
df = df.tail(self.window_size)
del df["tick_volume"]
scaler = MinMaxScaler(feature_range=(-1, 1))
for feat in ["open", "high", "low", "close"]:
tmp = df[feat].values.reshape((-1, 1))
scaler.partial_fit(tmp)
for feat in ["open", "high", "low", "close"]:
tmp = df[feat].values.reshape((-1, 1))
tmp = scaler.transform(tmp)
tmp = tmp.reshape((-1, ))
df[feat] = tmp
for feat in ["cdv"]:
tmp = df[feat].values.reshape((-1, 1))
tmp = MinMaxScaler(feature_range=(-1, 1)).fit_transform(tmp)
tmp = tmp.reshape((-1, ))
df[feat] = tmp
# PlotRender().plot_price_cdv(df)
# obs = obsList.values
obs = np.vstack([
df["open"].values,
df["cdv"].values,
df["high"].values,
df["cdv"].values,
df["low"].values,
df["cdv"].values,
df["close"].values,
df["cdv"].values,
])
obs = obs.reshape((1, ) + obs.shape + (1, ))
return obs
# get action from model using epsilon-greedy policy
def get_action(self, df):
df = df.copy()
df = df[["open", "high", "low", "close", "tick_volume"]]
x_i = self.get_heiken_cdv_obs_(df)
y_pred = self.model.predict(x_i)[0]
bear_proba = y_pred[0]
bull_proba = y_pred[1]
action = 0
"""proba_treshold = 0.55
if bull_proba > bear_proba or bull_proba > proba_treshold:
action = 0
elif bull_proba < bear_proba or bear_proba > proba_treshold:
action = 1
else:
action = 0"""
#if bull_proba > bear_proba and bull_proba > proba_treshold:
# action = 0
#else:
# action = 1
"""proba_treshold = 0.55
if bull_proba > proba_treshold:
action = 0
else:
action = 1"""
return action
def build_model(self):
pass
def fit_agent(self, env, nEpisodes, plotScores, saveFreq=5):
pass
def use_agent(self, env):
pass
def save_agent(self, path, name):
pass
def load_agent(self, path, name, dropSupportModel=False):
return self
def update_target_model(self):
pass
def loadPretrainedWeights(self, dir, baseName, agentType):
return self
# save sample <s,a,r,s'> to the replay memory
def append_sample(self, state, action, reward, next_state, done):
pass
# pick samples randomly from replay memory (with batch_size)
def train_model(self):
pass
class FeatGenForRL():
def __init__(self,
featureList,
nPoints=256,
window_size=64,
ema_period_volume=14,
ema_period_price=3):
self.nPoints = nPoints
self.featureList = featureList
self.window_size = window_size
self.ema_period_volume = ema_period_volume
self.ema_period_price = ema_period_price
self.fg_sw = FeatGen_ScaledWindow(
["open", "high", "low", "close", "cdv"],
nPoints=self.window_size,
nDiffs=0,
flatStack=False)
self.fg_cdv = FeatGen_CDV()
self.fg_pe = FeatGen_PriceEMA()
self.featureShape = (self.window_size, 5)
pass
def globalFit(self, df):
pass
def getFeatByDatetime(self, datetimeStr, historyData, expandDims=True):
x = self.getManyPointsFeat(datetimeStr, historyData)
return x
def getManyPointsFeat(self, ind, historyData):
df = historyData.copy()
df = df[["open", "close", "low", "high", "tick_volume"]]
#obs = self.get_price_ema_obs_(ind, df)
obs = self.get_heiken_cdv_obs_(ind, df)
return obs
def get_price_ema_obs_(self, ind, df):
df = df.loc[:ind]
df = df.tail(self.nPoints).copy()
df = self.fg_cdv.transform(df, self.ema_period_volume)
df = df.iloc[self.ema_period_volume - 1:]
df = self.fg_pe.transform(df, self.ema_period_price, verbose=False)
df = df.iloc[self.ema_period_price - 1:]
del df["tick_volume"]
ind = df.index.values[-1]
obs = self.fg_sw.get_window(ind, df)
return obs
def get_heiken_cdv_obs_(self, ind, df):
df = df.loc[:ind]
df = df.tail(self.nPoints).copy()
df = FeatGen_HeikenAshi().transform(df, verbose=False)
df = self.fg_cdv.transform(df, self.ema_period_volume)
df = df.iloc[self.ema_period_volume - 1:]
df = df.tail(self.window_size)
del df["tick_volume"]
scaler = MinMaxScaler(feature_range=(-1, 1))
for feat in ["open", "high", "low", "close"]:
tmp = df[feat].values.reshape((-1, 1))
scaler.partial_fit(tmp)
for feat in ["open", "high", "low", "close"]:
tmp = df[feat].values.reshape((-1, 1))
tmp = scaler.transform(tmp)
tmp = tmp.reshape((-1, ))
df[feat] = tmp
for feat in ["cdv"]:
tmp = df[feat].values.reshape((-1, 1))
tmp = MinMaxScaler(feature_range=(-1, 1)).fit_transform(tmp)
tmp = tmp.reshape((-1, ))
df[feat] = tmp
#PlotRender().plot_price_cdv(df)
obs = df.values
obs = obs.reshape((1, ) + obs.shape + (1, ))
return obs
def getMinDate(self, df):
minDate = df.index.values[self.nPoints]
return minDate
symbol = "EURUSD_i"
timeframe = "M5"
target_columns = ["open", "high", "low", "close", "tick_volume"]
window_size = 64
ema_period_volume = 14
ema_period_price = 3
dataManager = SymbolDataManager("../data/raw/")
df = dataManager.getData(symbol, timeframe)
df = df[target_columns]
df = df.tail(1000)
df = FeatGen_HeikenAshi().transform(df)
df = FeatGen_CDV().transform(df, ema_period_volume)
df = df.iloc[ema_period_volume - 1:]
#df = FeatGen_PriceEMA().transform(df, ema_period_price, verbose=False)
#df = df.iloc[ema_period_price - 1:]
#df = FeatGen_CDV().transform(df, ema_period_volume)
#df = df.iloc[ema_period_volume - 1:]
del df["tick_volume"]
df_vals = df.values
print(df.shape)
model = ResnetBuilder().build_resnet_34((1, 64, 5),
num_outputs=2,
m_cdv_feats = 44
use_heiken_ashi = True
batch_size = 128
model = load("../models/ex_stack.pkl")
#encoder = DenseAutoEncoder().loadEncoder( "../models/", "encoder" )
change_predictor = ChangePredictor(use_heiken_ashi, ema_period, n_price_feats,
m_cdv_feats, model, None, batch_size)
pred_probas = []
for i in tqdm(range(len(df_vals) - window_size),
desc="generating dataset",
colour="green"):
x_i = df_vals[i:i + window_size]
x_i = pd.DataFrame(x_i, columns=target_columns)
y_pred = change_predictor.predict_proba(x_i)
y_pred = list(y_pred)
pred_probas.append(y_pred)
stub_vals = [[0.0, 0.0] for i in range(window_size)]
pred_probas = stub_vals + pred_probas
pred_probas = np.array(pred_probas)
df["assurance_bull"] = pred_probas[:, 1]
df["assurance_bear"] = pred_probas[:, 0]
df = FeatGen_CDV().transform(df, period=ema_period)
PlotRender().plot_assurance(df)
print("done")
def proc_symbol_timeframe_pair( symbol, timeframe ):
feat_window_size = 64
ema_period_volume = 14
ema_period_price = 3
close_delta_window = 6
feat_gen = FeatGen_ScaledWindow(["open", "high", "low", "close", "cdv"], nPoints=feat_window_size, nDiffs=0, flatStack=False)
dataManager = SymbolDataManager("../data/raw/")
print(symbol + ": " + timeframe)
df = dataManager.getData(symbol, timeframe)
df = df[target_columns]
df = FeatGen_HeikenAshi().transform(df)
df = FeatGen_CDV().transform(df, ema_period_volume)
df = df.iloc[ema_period_volume - 1:]
#df = FeatGen_PriceEMA().transform(df, ema_period_price, verbose=False)
#df = df.iloc[ema_period_price - 1:]
#df = FeatGen_CDV().transform(df, ema_period_volume)
#df = df.iloc[ema_period_volume - 1:]
del df["tick_volume"]
###############################
df_vals = df.values
change_deltas = []
for i in range(1, len(df_vals) - close_delta_window):
prev_candle = df_vals[i - 1]
mean_delta = []
for j in range(close_delta_window):
current_candle = df_vals[i + j]
mean_delta.append( abs(current_candle[3] - prev_candle[3]) )
mean_delta = np.average( mean_delta )
change_deltas.append(mean_delta)
# change_treshold = np.percentile(change_deltas, change_percentile)
change_deltas = pd.Series(change_deltas)
quantiles = pd.qcut(change_deltas, 20, retbins=True)[1]
################################
x = []
y = []
for i in tqdm(range(df.index.values[0] + 1, df.index.values[-1] - feat_window_size - close_delta_window - 1), desc="generating dataset",
colour="green"):
current_candle = df.loc[i + feat_window_size]
mean_delta = []
for j in range(close_delta_window):
next_candle = df.loc[i + j + feat_window_size + 1]
mean_delta.append(next_candle[3] - current_candle[3])
mean_delta = np.average(mean_delta)
current_delta_sign = np.sign(mean_delta)
abs_delta = abs(mean_delta)
# current_delta = abs(((current_candle[3] - prev_candle[3]) + (current_candle[0] - prev_candle[0])) / 2.0)
min_delta_treshold = quantiles[-7] #70%
max_delta_treshold = quantiles[-2] #95%
if abs_delta < min_delta_treshold:
y_i = 0
elif abs_delta > min_delta_treshold and abs_delta < max_delta_treshold and current_delta_sign == -1:
y_i = -1
elif abs_delta > min_delta_treshold and abs_delta < max_delta_treshold and current_delta_sign == 1:
y_i = 1
else:
y_i = None
if y_i in [-1, 1]:
y.append(y_i)
ind = i + feat_window_size
######
# DEBUG
#ind += 1
######
x_i = feat_gen.get_window(ind, df)
x.append(x_i)
x = np.vstack(x)
y = np.array(y)
print(x.shape)
save(x, "../data/model_datasets/x_{}_{}.pkl".format(symbol, timeframe))
save(y, "../data/model_datasets/y_{}_{}.pkl".format(symbol, timeframe))
from classes.delta_bender.SymbolDataManager import SymbolDataManager
from classes.delta_bender.FeatGen_HeikenAshi import FeatGen_HeikenAshi
from classes.delta_bender.FeatGen_CDV import FeatGen_CDV
from classes.delta_bender import PlotRender
symbol = "EURUSD_i"
timeframe = "M15"
dataManager = SymbolDataManager("../data/raw/")
df = dataManager.getData(symbol, timeframe)
print(df)
df = df.tail(1200)
df = FeatGen_HeikenAshi().transform(df, verbose=True)
#PlotRender().plot_candles(df)
df = FeatGen_CDV().transform(df, period=32, verbose=True)
#PlotRender().plot_cdv(df)
PlotRender().plot_price_cdv(df)
print("done")
current_mean = (current_candle[3] + current_candle[0]) / 2.0
mean_delta = current_mean - prev_mean
if abs(mean_delta) < change_treshold:
y_i = 0
elif mean_delta < 0:
y_i = -1
else:
y_i = 1
if y_i in [-1, 1]:
#########
tmp = df_vals[i + 1:i + window_size + 2]
tmp = pd.DataFrame(tmp, columns=target_columns)
tmp = FeatGen_CDV().transform(tmp, period=64)
PlotRender().plot_price_cdv(tmp)
#########
y.append(y_i)
x_i = df_vals[i + 1:i + window_size + 1]
x_i = feat_gen.transform(x_i,
ema_period=ema_period,
n_price_feats=n_price_feats,
m_cdv_feats=m_cdv_feats)
x.append(x_i)
x = np.array(x)
y = np.array(y)
print(x.shape)
class FeatGenMeta():
def __init__(self, use_heiken_ashi):
self.needful_cols = ["open", "high", "low", "close", "tick_volume"]
if use_heiken_ashi:
self.gen_hk = FeatGen_HeikenAshi()
else:
self.gen_hk = None
self.gen_cdv = FeatGen_CDV()
pass
def transform(self, x, ema_period, n_price_feats=None, m_cdv_feats=None):
if isinstance(x, np.ndarray):
x = pd.DataFrame(x, columns=self.needful_cols)
elif isinstance(x, pd.DataFrame):
x.reset_index(drop=True, inplace=True)
if self.gen_hk is not None:
x = self.gen_hk.transform(x)
x = self.gen_cdv.transform(x, period=ema_period, verbose=False)
x = x[ema_period - 1:]
x.reset_index(drop=True, inplace=True)
cdv_vals = x["cdv"].values
cdv_change_ind = []
for i in range(1, len(cdv_vals)):
cdv_change_ind.append(cdv_vals[i] - cdv_vals[i - 1])
"""if cdv_vals[i] > cdv_vals[i-1]:
cdv_change_ind.append( 1 )
elif cdv_vals[i] < cdv_vals[i-1]:
cdv_change_ind.append( -1 )
else:
cdv_change_ind.append( 0 )"""
cdv_change_ind = np.array(cdv_change_ind).reshape((-1, 1))
cdv_change_ind = MinMaxScaler(
feature_range=(-1, 1)).fit_transform(cdv_change_ind)
cdv_change_ind = np.array(cdv_change_ind).reshape((-1, ))
x = x[1:]
x.reset_index(drop=True, inplace=True)
x["cdv_change_ind"] = cdv_change_ind
cdv_vals = np.array(cdv_vals).reshape((-1, 1))
cdv_vals = MinMaxScaler(feature_range=(-1, 1)).fit_transform(cdv_vals)
cdv_vals = np.array(cdv_vals).reshape((-1, ))
if m_cdv_feats is not None:
cdv_vals = cdv_vals[-m_cdv_feats:]
cdv_change_feats = x["cdv_change_ind"].values[-m_cdv_feats:]
else:
cdv_change_feats = x["cdv_change_ind"].values
price_feats = self.get_price_feats_(x, n_price_feats)
feats = np.hstack([price_feats, cdv_vals, cdv_change_feats])
return feats
def get_price_feats_(self, df, n):
df = df.copy()
if n is not None:
df = df.tail(n)
a = np.array([i for i in range(n)]).reshape((-1, 1))
x = np.vstack([a, a, a, a])
y = np.hstack([
df["open"].values, df["high"].values, df["low"].values,
df["close"].values
])
y = np.array(y).reshape((-1, 1))
y = MinMaxScaler(feature_range=(-1, 1)).fit_transform(y)
y = np.array(y).reshape((-1, ))
model = Ridge(random_state=45).fit(x, y)
y_pred = model.predict(a)
scaled_prices = []
for i in range(4):
price_property = y[i * n:(i + 1) * n]
scaled_prices.append(price_property)
scaled_prices = np.array(scaled_prices)
price_deltas = scaled_prices - y_pred
means = np.mean(price_deltas, axis=0)
stds = np.std(price_deltas, axis=0)
min_deltas = np.min(price_deltas, axis=0)
max_deltas = np.max(price_deltas, axis=0)
std_global = np.std(price_deltas)
min_deltas_global = np.min(price_deltas)
max_deltas_global = np.max(price_deltas)
angle = self.get_angle_(0.0, y_pred[0], 1.0, y_pred[1])
price_feats = np.hstack([
y, means, stds, min_deltas, max_deltas,
[std_global, min_deltas_global, max_deltas_global, angle]
])
return price_feats
def get_angle_(self, x_1, y_1, x_2, y_2):
x = x_2 - x_1
y = y_2 - y_1
cos_x = x / np.sqrt(np.square(x) + np.square(y))
sin_x = np.sqrt(1 - np.square(cos_x))
return sin_x