def add_features_category(df: pd.DataFrame,
momentum=False,
volatility=False,
trend=False,
drop_times=True):
df['h'] = pd.to_numeric(df['datetime'].str[-9:-7])
df['min'] = pd.to_numeric(df['datetime'].str[-6:-4])
df['year'] = pd.to_numeric(df['datetime'].str[0:4])
if drop_times:
df = df.drop("datetime", 1)
if momentum:
df = df.astype('float32')
ta.add_momentum_ta(df,
high='hi',
low='lo',
close='last',
volume='vl',
fillna=False)
if trend:
df = df.astype('float32')
ta.add_trend_ta(df, high='hi', low='lo', close='last', fillna=False)
if volatility:
df = df.astype('float32')
ta.add_volatility_ta(df,
high='hi',
low='lo',
close='last',
fillna=False)
return df
def _feature_engineering(df, volume=True):
if volume:
df = ta.add_all_ta_features(df,
open="Open",
high="High",
low="Low",
close="Last",
volume="Volume",
fillna=False)
else:
df = ta.add_momentum_ta(df,
high="High",
low="Low",
close="Last",
volume="Volume",
fillna=False)
df = ta.add_volatility_ta(df,
high="High",
low="Low",
close="Last",
fillna=False)
df = ta.add_trend_ta(df,
high="High",
low="Low",
close="Last",
fillna=False)
df = ta.add_others_ta(df, close="Last", fillna=False)
df["trend_psar_up"] = df["trend_psar_up"].fillna(0.0)
df["trend_psar_down"] = df["trend_psar_down"].fillna(0.0)
return df
def add_technical_indicators_trend(stock_dataframe: pd.DataFrame,
open="Open",
high="High",
low="Low",
close="Close",
volume="Volume") -> pd.DataFrame:
return add_trend_ta(df=stock_dataframe, high=high, low=low, close=close)
def process_one(df):
df = utils.dropna(df)
if args.custom_indicator_fn is not None:
if args.custom_indicator is not None:
raise ValueError('custom indicator fn and custom indicator only can use one')
# print(args.custom_indicator_fn)
df = add_custom_ta(df, args.custom_indicator_fn + '(' + args.custom_indicator_args + ')')
elif args.custom_indicator is not None:
df = add_custom_ta(df, args.custom_indicator)
elif args.indicator == 'all':
df = add_all_ta(df, "open", "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'trend':
df = add_trend_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'momentum':
df = add_momentum_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'my':
df = add_my_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'volatility':
df = add_volatility_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'volume':
df = add_volume_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
else:
raise ValueError('Please use a calculation')
return df
def getLength():
coin = "BAT"
df = fetchData(symbol=(coin + "USDT"), amount=2, timeframe='4h')
# Drop unix and set 'date' as index
df = df[['date', 'open', 'high', 'low', 'close',
'volume']] #5 indicators (date not included in count)
df = df.set_index("date")
test = df.copy()
fresh1 = df.copy()
fresh2 = df.copy()
fresh3 = df.copy()
fresh4 = df.copy()
momentum = ta.add_momentum_ta(fresh1,
high='high',
low='low',
close='close',
volume='volume')
volume = ta.add_volume_ta(fresh2,
high='high',
low='low',
close='close',
volume='volume')
trend = ta.add_trend_ta(fresh3, high='high', low='low', close='close')
volatility = ta.add_volatility_ta(fresh4,
high='high',
low='low',
close='close')
print("Amount of momentum indicators: " + str(len(momentum.columns) - 5))
print("Amount of volume indicators: " + str(len(volume.columns) - 5))
print("Amount of trend indicators: " + str(len(trend.columns) - 5))
print("Amount of volatility indicators: " +
str(len(volatility.columns) - 5))
def get_data_bars(symbols, timeframe='5Min'):
data = pyt.client.get_barset(symbols, timeframe, limit=50).df
for symbol in symbols:
feats = add_momentum_ta(data.get(symbol), 'high', 'low', 'close',
'volume')
feats = add_trend_ta(feats, 'high', 'low', 'close')
for f in feats:
data.loc[:, (symbol, f)] = feats.loc[:, f]
return data
def get_trend_indicators(df, threshold=0.5, plot=False):
df_trend = df.copy()
# add custom trend indicators
df_trend["sma7"] = SMAIndicator(close=df["Close"], window=7, fillna=True).sma_indicator()
df_trend["sma25"] = SMAIndicator(close=df["Close"], window=25, fillna=True).sma_indicator()
df_trend["sma99"] = SMAIndicator(close=df["Close"], window=99, fillna=True).sma_indicator()
df_trend = add_trend_ta(df_trend, high="High", low="Low", close="Close")
return DropCorrelatedFeatures(df_trend, threshold, plot)
def get_historical_data(*, logger, client, symbol, interval):
"""
Return historical pricing data from the Binance Exchange formatted for use.
Parameters:
logger (logging.Logger): An open logging object.
client (binance.client.Client): An open connected client to the Binance Exchange API.
symbol (str): A Crypto-Pair symbol.
interval (str): An OHLC candlestick width.
Returns:
pandas.core.frame.DataFrame: Historical pricing data for the given symbol and interval.
"""
data = util.binance.get_historical_data(client=client,
symbol=symbol,
interval=interval)
ta.add_momentum_ta(data,
'high',
'low',
'close',
'volume',
fillna=True,
colprefix='')
ta.add_trend_ta(data, 'high', 'low', 'close', fillna=True, colprefix='')
ta.add_volatility_ta(data,
'high',
'low',
'close',
fillna=True,
colprefix='')
data = util.db.organize_table_history(data=data)
return data
def get_features(df, options=None):
if options == 'all':
df = ta.add_all_ta_features(df, "open", "high", "low", "close",
"volume")
elif options == 'volume':
df = ta.add_volume_ta(df, "high", "low", "close", "volume")
elif options == 'momentum':
df = ta.add_momentum_ta(df, "high", "low", "close", "volume")
elif options == 'volatility':
df = ta.add_volatility_ta(df, "high", "low", "close")
elif options == 'trend':
df = ta.add_trend_ta(df, "high", "low", "close")
elif options == 'others':
df = ta.add_others_ta(df, "close")
df = df.sort_values('date')
return df
def add_custom_ta_features(
df: pd.DataFrame,
open: str, # noqa
high: str,
low: str,
close: str,
fillna: bool = False,
colprefix: str = "",
apply_pct: bool = False,
plot: bool = False,
) -> pd.DataFrame:
# Add Volatility TA
df = ta.add_volatility_ta(df=df,
high=high,
low=low,
close=close,
fillna=fillna,
colprefix=colprefix)
# Add Trend TA
df = ta.add_trend_ta(df=df,
high=high,
low=low,
close=close,
fillna=fillna,
colprefix=colprefix)
# Add Other TA
df = ta.add_others_ta(df=df,
close=close,
fillna=fillna,
colprefix=colprefix)
# convert to pct
if apply_pct:
df = df.pct_change(fill_method='ffill')
df = df.applymap(lambda x: x * 100)
df.replace([np.inf, -np.inf], np.nan, inplace=True)
df.astype(np.float32)
df = df.round(5)
if fillna:
df.fillna(value=0, inplace=True)
if plot:
fig = make_subplots(rows=5,
cols=1,
shared_xaxes=True,
vertical_spacing=0.02,
subplot_titles=("Close", "Bollinger Bands",
"MACD"))
fig.add_trace(go.Scatter(x=df.index, y=df['close'], name=symbol),
row=1,
col=1)
# Bollinger-Bands
fig.add_trace(go.Scatter(x=df.index, y=df['close'], name=symbol),
row=2,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['volatility_bbh'],
name=symbol + ' High BB'),
row=2,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['volatility_bbl'],
name=symbol + ' Low BB'),
row=2,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['volatility_bbm'],
name=symbol + ' EMA BB'),
row=2,
col=1)
# MACD
fig.add_trace(go.Scatter(x=df.index,
y=df['trend_macd'],
name=symbol + ' MACD'),
row=3,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['trend_macd_signal'],
name=symbol + ' MACD Signal'),
row=3,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['trend_macd_diff'],
name=symbol + ' MACD Difference'),
row=3,
col=1)
# SMA
fig.add_trace(go.Scatter(x=df.index, y=df['close'], name=symbol),
row=4,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['trend_sma_fast'],
name=symbol + ' SMA-Fast'),
row=4,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['trend_sma_slow'],
name=symbol + ' SMA-Slow'),
row=4,
col=1)
# EMA
fig.add_trace(go.Scatter(x=df.index, y=df['close'], name=symbol),
row=5,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['trend_ema_fast'],
name=symbol + ' EMA-Fast'),
row=5,
col=1)
fig.add_trace(go.Scatter(x=df.index,
y=df['trend_ema_slow'],
name=symbol + ' EMA-Slow'),
row=5,
col=1)
config = {'displayModeBar': False}
fig.show(config=config)
return df
def EDA(file, debug=None, tech_analysis=False, start_date=None, end_date=None):
if not file:
raise "No file provided."
# Load file
df = pd.read_csv(file, index_col='Date')
if debug:
print(f"Exploratory data analysis for {file.split('/')[-1]}")
# Some details on the type of data
print(df.info())
print(df.describe())
print(df.head())
# Check for dataframe containing NaN values
print(f"Any cell containing NaN value: {df.isnull().values.any()}")
# Plot open and close prices
ax = plt.gca()
df.plot(y='Open', ax=ax, grid=True)
df.plot(y='Close', ax=ax, grid=True)
plt.show()
""" Introduce new attributes
Target is close/open - 1 for regression task and Up_down for
classification
"""
df['Close_close'] = (df['Close'] / df['Close'].shift(1)) - 1
df['High_high'] = (df['High'] / df['High'].shift(1)) - 1
df['Low_low'] = (df['Low'] / df['Low'].shift(1)) - 1
df['Close_open'] = df['Close'] / df['Open'] - 1
df['Up_down'] = np.sign(df['Close_open']).astype('int')
df['High_low'] = df['High'] / df['Low'] - 1
df['Close_open-1'] = df['Close_open'].shift(1)
df['Log_volume'] = df['Volume'].apply(np.log)
if debug:
# Plot it
df.plot(y='Close_close')
plt.show()
df.plot(y='High_high')
plt.show()
df.plot(y='Low_low')
plt.show()
df.plot(y='Close_open')
plt.show()
df.plot(y='High_low')
plt.show()
# Add all Technical analysis features if variable ta is true
if tech_analysis:
#df = ta.add_all_ta_features(df, 'Open', 'High', 'Low', 'Close', 'Volume',
# fillna=True)
df = ta.add_volume_ta(df, 'Open', 'High', 'Low', 'Close', 'Volume')
#df = ta.add_momentum_ta(df, 'Open', 'High', 'Low', 'Close', 'Volume')
df = ta.add_volatility_ta(df, 'Open', 'High', 'Low', 'Close', 'Volume')
df = ta.add_trend_ta(df, 'Open', 'High', 'Low', 'Close', 'Volume')
if debug:
print(df.info())
print(f"Starting date = {start_date}")
print(f"Ending date = {end_date}")
plt.plot(df[start_date:end_date].Close)
plt.plot(df[start_date:end_date].volatility_bbh, label='High BB')
plt.plot(df[start_date:end_date].volatility_bbl, label='Low BB')
plt.plot(df[start_date:end_date].volatility_bbm, label='EMA BB')
plt.title('Bollinger bands')
plt.legend()
plt.show()
return df
def AddFeatures(data, category):
"""Add feature to the data.
Args:
--
data : (DataFrame)
The DataFrame of the stocks.
category : (str)
Composed of 5 arguments.
"all" : Append all features.
"trend": Append all trend-related features.
"volatility": Append all volatile-related features.
"volume": Append all volume-related features.
"emaribbon": Append all ema + volume features.
"oscillators": Append all oscillators + volume features.
Returns:
pandas.core.frame.DataFrame: Dataframe with new features.
"""
# Store list of tradable stocks
if len(data["stock code"]) > 0:
data.drop("stock code", 1, inplace=True) # drop stock code
data = data.apply(pd.to_numeric)
if category == "all" or category == "momentum":
# Append all Momentum-related features
print(f'data ${data}')
data = ta.add_momentum_ta(data,
high="High",
low="Low",
close="Close",
volume="Volume")
elif category == "all" or category == "trend":
# Append all tren-related features
data = ta.add_trend_ta(data, high="High", low="Low", close="Close")
elif category == "all" or category == "volatility":
# Append all volatility-related features
data = ta.add_volatility_ta(data,
high="High",
low="Low",
close="Close")
elif category == "all" or category == "volume":
# Append all volume-related features
data = ta.add_volume_ta(data,
high="High",
low="Low",
close="Close",
volume="Volume")
elif category == "emaribbon":
# Append user-specified features
data = indicators.ema(data, period=9)
data = indicators.ema(data, period=20)
data = indicators.ema(data, period=25)
data = indicators.ema(data, period=30)
data = indicators.ema(data, period=35)
data = indicators.ema(data, period=40)
data = indicators.ema(data, period=45)
data = indicators.ema(data, period=50)
data = indicators.ema(data, period=55)
data = indicators.volume(data, period=20)
elif category == 'bottom_fish':
# BUY:
# CCI > -100
# DCHL/DCHLI: Candle must touch the the lower indicators0
data = ta.add_all_ta_features(data,
open='Open',
high='High',
low='Low',
close='Close',
volume='Volume')
bottom_fish_list = ["Open","High","Low","Close", "Volume", \
"volatility_dcl", "volatility_dch", "volatility_dcli", "volatility_dchi", "trend_cci"]
bottom_fish = data[bottom_fish_list]
bottom_fish['volatility_dcl'] = data['volatility_dcl']
bottom_fish['volatility_dch'] = data['volatility_dch']
bottom_fish['volatility_dcli'] = data['volatility_dcl']
bottom_fish['volatility_dchi'] = data['volatility_dch']
bottom_fish['trend_cci'] = data['trend_cci']
bottom_fish = indicators.volume(bottom_fish, period=20)
data = bottom_fish
elif category == 'macdo':
indicator_macd = MACD(close=data['Close'],
n_slow=20,
n_fast=5,
n_sign=5)
data['trend_macd'] = indicator_macd.macd()
data = indicators.ma(data, period=200)
data = indicators.ema(data, period=62)
data = indicators.ema(data, period=38)
data = indicators.volume(data, period=20)
elif category == 'almarsi':
alma_indicator = ALMAIndicator(close=data['Close'])
data['alma9'] = alma_indicator.alma()
# data['alma20'] = alma_indicator.alma_weights(window=20)
# data['alma50'] = alma_indicator.alma_weights(window=50)
data = indicators.rsi(data, period=9)
data = indicators.rsi(data, period=25)
data = indicators.rsi(data, period=12)
# print (data)
if category == "all":
data.dropna(inplace=True, axis=1)
else:
data.dropna(inplace=True)
# print (data)
# Scale the value
for col in data.columns[:]:
data[col] = preprocessing.scale(data[col].values)
# print (data)
return data
def transform(self, X, y=None) -> pd.DataFrame:
"""
Calculates features (technical indicators) of selected type.
Parameters
----------
X : iterable
Data in OHLCV format.
y : iterable, default=None
Training targets.
Returns
-------
Xt : array-like of shape (n_samples, n_transformed_features)
Dataset with calculated features.
"""
if self.all_features:
return add_all_ta_features(
df=X,
open="open",
high="high",
low="low",
close="close",
volume="volume",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.volume_features:
X = add_volume_ta(
df=X,
high="high",
low="low",
close="close",
volume="volume",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.volatility_features:
X = add_volatility_ta(
df=X,
high="high",
low="low",
close="close",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.trend_features:
X = add_trend_ta(
df=X,
high="high",
low="low",
close="close",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.momentum_features:
X = add_momentum_ta(
df=X,
high="high",
low="low",
close="close",
volume="volume",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.others_features:
X = add_others_ta(df=X,
close="close",
fillna=self.fillna,
colprefix=self.colprefix)
return X
def makePlot(type):
# Fetch the data
# Option 1: Use CryptoDataDownload
#cdd = CryptoDataDownload()
#df = cdd.fetch("Bitfinex", "USD", "BTC", "d")
# Option 2: Use Binance data
# Found here: https://github.com/StephanAkkerman/TensorTrade
coin = "BAT"
df = fetchData(symbol=(coin + "USDT"), amount=2, timeframe='4h')
# Drop unix and set 'date' as index
df = df[['date', 'open', 'high', 'low', 'close', 'volume']]
df = df.set_index("date")
# Calculate absolute correlation and clean up names
if (type == 'momentum'):
df = ta.add_momentum_ta(df,
high='high',
low='low',
close='close',
volume='volume').corr().abs()
df.columns = df.columns.str.replace('momentum_', '')
df.index = df.index.str.replace('momentum_', '')
if (type == 'volume'):
df = ta.add_volume_ta(df,
high='high',
low='low',
close='close',
volume='volume').corr().abs()
df.columns = df.columns.str.replace('volume_', '')
df.index = df.index.str.replace('volume_', '')
if (type == 'trend'):
df = ta.add_trend_ta(df, high='high', low='low',
close='close').corr().abs()
df.columns = df.columns.str.replace('trend_', '')
df.index = df.index.str.replace('trend_', '')
if (type == 'volatility'):
df = ta.add_volatility_ta(df, high='high', low='low',
close='close').corr().abs()
df.columns = df.columns.str.replace('volatility_', '')
df.index = df.index.str.replace('volatility_', '')
if (type == 'others'):
df = ta.add_others_ta(df, close='close').corr().abs()
df.columns = df.columns.str.replace('volumeothers_', '')
df.index = df.index.str.replace('volumeothers_', '')
if (type == 'all'):
df = ta.add_all_ta_features(df,
open='open',
high='high',
low='low',
close='close',
volume='volume').corr().abs()
# Remove this from heatmap
df = df.drop(index=['open', 'high', 'low', 'close', 'volume'],
columns=['open', 'high', 'low', 'close', 'volume'])
# Plot the heatmap
plt.subplots(figsize=(10, 10))
# https://seaborn.pydata.org/tutorial/color_palettes.html for possible colors
sns.heatmap(df,
annot=False,
linewidth=0,
center=0.5,
cmap=sns.color_palette("viridis", as_cmap=True))
plt.tight_layout()
plt.yticks(rotation=0)
plt.show()
