def add_features_category(df: pd.DataFrame,
momentum=False,
volatility=False,
trend=False,
drop_times=True):
df['h'] = pd.to_numeric(df['datetime'].str[-9:-7])
df['min'] = pd.to_numeric(df['datetime'].str[-6:-4])
df['year'] = pd.to_numeric(df['datetime'].str[0:4])
if drop_times:
df = df.drop("datetime", 1)
if momentum:
df = df.astype('float32')
ta.add_momentum_ta(df,
high='hi',
low='lo',
close='last',
volume='vl',
fillna=False)
if trend:
df = df.astype('float32')
ta.add_trend_ta(df, high='hi', low='lo', close='last', fillna=False)
if volatility:
df = df.astype('float32')
ta.add_volatility_ta(df,
high='hi',
low='lo',
close='last',
fillna=False)
return df
def load_data():
# Load financial dataset
df = pd.read_csv(
'https://raw.githubusercontent.com/bukosabino/streamlit-demo-financial-eda/master/data/data.csv',
sep=',')
# Clean NaN values
df = ta.utils.dropna(df)
# Apply feature engineering (technical analysis)
# df = ta.add_all_ta_features(df, "Open", "High", "Low", "Close", "Volume_BTC", fillna=True)
df = ta.add_volatility_ta(df,
"High",
"Low",
"Close",
fillna=False,
colprefix=ta_col_prefix)
df = ta.add_momentum_ta(df,
"High",
"Low",
"Close",
"Volume_Currency",
fillna=False,
colprefix=ta_col_prefix)
return df
def _feature_engineering(df, volume=True):
if volume:
df = ta.add_all_ta_features(df,
open="Open",
high="High",
low="Low",
close="Last",
volume="Volume",
fillna=False)
else:
df = ta.add_momentum_ta(df,
high="High",
low="Low",
close="Last",
volume="Volume",
fillna=False)
df = ta.add_volatility_ta(df,
high="High",
low="Low",
close="Last",
fillna=False)
df = ta.add_trend_ta(df,
high="High",
low="Low",
close="Last",
fillna=False)
df = ta.add_others_ta(df, close="Last", fillna=False)
df["trend_psar_up"] = df["trend_psar_up"].fillna(0.0)
df["trend_psar_down"] = df["trend_psar_down"].fillna(0.0)
return df
def process_one(df):
df = utils.dropna(df)
if args.custom_indicator_fn is not None:
if args.custom_indicator is not None:
raise ValueError('custom indicator fn and custom indicator only can use one')
# print(args.custom_indicator_fn)
df = add_custom_ta(df, args.custom_indicator_fn + '(' + args.custom_indicator_args + ')')
elif args.custom_indicator is not None:
df = add_custom_ta(df, args.custom_indicator)
elif args.indicator == 'all':
df = add_all_ta(df, "open", "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'trend':
df = add_trend_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'momentum':
df = add_momentum_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'my':
df = add_my_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'volatility':
df = add_volatility_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'volume':
df = add_volume_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
else:
raise ValueError('Please use a calculation')
return df
def getLength():
coin = "BAT"
df = fetchData(symbol=(coin + "USDT"), amount=2, timeframe='4h')
# Drop unix and set 'date' as index
df = df[['date', 'open', 'high', 'low', 'close',
'volume']] #5 indicators (date not included in count)
df = df.set_index("date")
test = df.copy()
fresh1 = df.copy()
fresh2 = df.copy()
fresh3 = df.copy()
fresh4 = df.copy()
momentum = ta.add_momentum_ta(fresh1,
high='high',
low='low',
close='close',
volume='volume')
volume = ta.add_volume_ta(fresh2,
high='high',
low='low',
close='close',
volume='volume')
trend = ta.add_trend_ta(fresh3, high='high', low='low', close='close')
volatility = ta.add_volatility_ta(fresh4,
high='high',
low='low',
close='close')
print("Amount of momentum indicators: " + str(len(momentum.columns) - 5))
print("Amount of volume indicators: " + str(len(volume.columns) - 5))
print("Amount of trend indicators: " + str(len(trend.columns) - 5))
print("Amount of volatility indicators: " +
str(len(volatility.columns) - 5))
def get_data_bars(symbols, timeframe='5Min'):
data = pyt.client.get_barset(symbols, timeframe, limit=50).df
for symbol in symbols:
feats = add_momentum_ta(data.get(symbol), 'high', 'low', 'close',
'volume')
feats = add_trend_ta(feats, 'high', 'low', 'close')
for f in feats:
data.loc[:, (symbol, f)] = feats.loc[:, f]
return data
def get_momentum_indicators(df, threshold=0.5, plot=False):
df_momentum = df.copy()
# add custom momentum indicators
# ...
df_momentum = add_momentum_ta(df_momentum, high="High", low="Low", close="Close", volume="Volume")
return DropCorrelatedFeatures(df_momentum, threshold, plot)
def add_technical_indicators_momentum(stock_dataframe: pd.DataFrame,
open="Open",
high="High",
low="Low",
close="Close",
volume="Volume") -> pd.DataFrame:
return add_momentum_ta(df=stock_dataframe,
high=high,
low=low,
close=close,
volume=volume)
def get_historical_data(*, logger, client, symbol, interval):
"""
Return historical pricing data from the Binance Exchange formatted for use.
Parameters:
logger (logging.Logger): An open logging object.
client (binance.client.Client): An open connected client to the Binance Exchange API.
symbol (str): A Crypto-Pair symbol.
interval (str): An OHLC candlestick width.
Returns:
pandas.core.frame.DataFrame: Historical pricing data for the given symbol and interval.
"""
data = util.binance.get_historical_data(client=client,
symbol=symbol,
interval=interval)
ta.add_momentum_ta(data,
'high',
'low',
'close',
'volume',
fillna=True,
colprefix='')
ta.add_trend_ta(data, 'high', 'low', 'close', fillna=True, colprefix='')
ta.add_volatility_ta(data,
'high',
'low',
'close',
fillna=True,
colprefix='')
data = util.db.organize_table_history(data=data)
return data
def get_features(df, options=None):
if options == 'all':
df = ta.add_all_ta_features(df, "open", "high", "low", "close",
"volume")
elif options == 'volume':
df = ta.add_volume_ta(df, "high", "low", "close", "volume")
elif options == 'momentum':
df = ta.add_momentum_ta(df, "high", "low", "close", "volume")
elif options == 'volatility':
df = ta.add_volatility_ta(df, "high", "low", "close")
elif options == 'trend':
df = ta.add_trend_ta(df, "high", "low", "close")
elif options == 'others':
df = ta.add_others_ta(df, "close")
df = df.sort_values('date')
return df
def load_ohlcv_data(symbol, dates):
# Load ohlc data
df = load_data_ohlcv(symbol, dates, dirname=DIRNAME)
# Apply technical analysis
df = ta.add_volatility_ta(df,
"high",
"low",
"close",
fillna=False,
colprefix="ta_")
df = ta.add_momentum_ta(df,
"high",
"low",
"close",
"volume",
fillna=False,
colprefix="ta_")
df = add_custom_trend(df, "close", fillna=False, colprefix="ta_")
return df
def create_features(stock):
# Store list of tradable stocks
data = load_data().get_group(stock)
data.drop("Netforeign",1,inplace=True) #drop netforeign
data.drop("stock code",1,inplace=True) #drop stock code
# Convert all columns data frame to numeric columns
data = data.apply(pd.to_numeric)
data = ta.add_momentum_ta(data, high="High", low="Low", close="Close", volume="Volume")
# Remove all NaN value
data.dropna(inplace=True)
# Standardize the data
for col in data.columns[:]:
data[col] = preprocessing.scale(data[col].values)
# Create the target variable that take the values of 1 if the stock price go up or -1 if the stock price go down
target = np.where(data['Close'].shift(-1) > data['Close'], 1, -1)
return data, target
def addAllTa(df):
df = ta.add_momentum_ta(df, 'open', 'high', 'low', 'close', 'volume', True)
return df
def AddFeatures(data, category):
"""Add feature to the data.
Args:
--
data : (DataFrame)
The DataFrame of the stocks.
category : (str)
Composed of 5 arguments.
"all" : Append all features.
"trend": Append all trend-related features.
"volatility": Append all volatile-related features.
"volume": Append all volume-related features.
"emaribbon": Append all ema + volume features.
"oscillators": Append all oscillators + volume features.
Returns:
pandas.core.frame.DataFrame: Dataframe with new features.
"""
# Store list of tradable stocks
if len(data["stock code"]) > 0:
data.drop("stock code", 1, inplace=True) # drop stock code
data = data.apply(pd.to_numeric)
if category == "all" or category == "momentum":
# Append all Momentum-related features
print(f'data ${data}')
data = ta.add_momentum_ta(data,
high="High",
low="Low",
close="Close",
volume="Volume")
elif category == "all" or category == "trend":
# Append all tren-related features
data = ta.add_trend_ta(data, high="High", low="Low", close="Close")
elif category == "all" or category == "volatility":
# Append all volatility-related features
data = ta.add_volatility_ta(data,
high="High",
low="Low",
close="Close")
elif category == "all" or category == "volume":
# Append all volume-related features
data = ta.add_volume_ta(data,
high="High",
low="Low",
close="Close",
volume="Volume")
elif category == "emaribbon":
# Append user-specified features
data = indicators.ema(data, period=9)
data = indicators.ema(data, period=20)
data = indicators.ema(data, period=25)
data = indicators.ema(data, period=30)
data = indicators.ema(data, period=35)
data = indicators.ema(data, period=40)
data = indicators.ema(data, period=45)
data = indicators.ema(data, period=50)
data = indicators.ema(data, period=55)
data = indicators.volume(data, period=20)
elif category == 'bottom_fish':
# BUY:
# CCI > -100
# DCHL/DCHLI: Candle must touch the the lower indicators0
data = ta.add_all_ta_features(data,
open='Open',
high='High',
low='Low',
close='Close',
volume='Volume')
bottom_fish_list = ["Open","High","Low","Close", "Volume", \
"volatility_dcl", "volatility_dch", "volatility_dcli", "volatility_dchi", "trend_cci"]
bottom_fish = data[bottom_fish_list]
bottom_fish['volatility_dcl'] = data['volatility_dcl']
bottom_fish['volatility_dch'] = data['volatility_dch']
bottom_fish['volatility_dcli'] = data['volatility_dcl']
bottom_fish['volatility_dchi'] = data['volatility_dch']
bottom_fish['trend_cci'] = data['trend_cci']
bottom_fish = indicators.volume(bottom_fish, period=20)
data = bottom_fish
elif category == 'macdo':
indicator_macd = MACD(close=data['Close'],
n_slow=20,
n_fast=5,
n_sign=5)
data['trend_macd'] = indicator_macd.macd()
data = indicators.ma(data, period=200)
data = indicators.ema(data, period=62)
data = indicators.ema(data, period=38)
data = indicators.volume(data, period=20)
elif category == 'almarsi':
alma_indicator = ALMAIndicator(close=data['Close'])
data['alma9'] = alma_indicator.alma()
# data['alma20'] = alma_indicator.alma_weights(window=20)
# data['alma50'] = alma_indicator.alma_weights(window=50)
data = indicators.rsi(data, period=9)
data = indicators.rsi(data, period=25)
data = indicators.rsi(data, period=12)
# print (data)
if category == "all":
data.dropna(inplace=True, axis=1)
else:
data.dropna(inplace=True)
# print (data)
# Scale the value
for col in data.columns[:]:
data[col] = preprocessing.scale(data[col].values)
# print (data)
return data
def transform(self, X, y=None) -> pd.DataFrame:
"""
Calculates features (technical indicators) of selected type.
Parameters
----------
X : iterable
Data in OHLCV format.
y : iterable, default=None
Training targets.
Returns
-------
Xt : array-like of shape (n_samples, n_transformed_features)
Dataset with calculated features.
"""
if self.all_features:
return add_all_ta_features(
df=X,
open="open",
high="high",
low="low",
close="close",
volume="volume",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.volume_features:
X = add_volume_ta(
df=X,
high="high",
low="low",
close="close",
volume="volume",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.volatility_features:
X = add_volatility_ta(
df=X,
high="high",
low="low",
close="close",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.trend_features:
X = add_trend_ta(
df=X,
high="high",
low="low",
close="close",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.momentum_features:
X = add_momentum_ta(
df=X,
high="high",
low="low",
close="close",
volume="volume",
fillna=self.fillna,
colprefix=self.colprefix,
)
if self.others_features:
X = add_others_ta(df=X,
close="close",
fillna=self.fillna,
colprefix=self.colprefix)
return X
def makePlot(type):
# Fetch the data
# Option 1: Use CryptoDataDownload
#cdd = CryptoDataDownload()
#df = cdd.fetch("Bitfinex", "USD", "BTC", "d")
# Option 2: Use Binance data
# Found here: https://github.com/StephanAkkerman/TensorTrade
coin = "BAT"
df = fetchData(symbol=(coin + "USDT"), amount=2, timeframe='4h')
# Drop unix and set 'date' as index
df = df[['date', 'open', 'high', 'low', 'close', 'volume']]
df = df.set_index("date")
# Calculate absolute correlation and clean up names
if (type == 'momentum'):
df = ta.add_momentum_ta(df,
high='high',
low='low',
close='close',
volume='volume').corr().abs()
df.columns = df.columns.str.replace('momentum_', '')
df.index = df.index.str.replace('momentum_', '')
if (type == 'volume'):
df = ta.add_volume_ta(df,
high='high',
low='low',
close='close',
volume='volume').corr().abs()
df.columns = df.columns.str.replace('volume_', '')
df.index = df.index.str.replace('volume_', '')
if (type == 'trend'):
df = ta.add_trend_ta(df, high='high', low='low',
close='close').corr().abs()
df.columns = df.columns.str.replace('trend_', '')
df.index = df.index.str.replace('trend_', '')
if (type == 'volatility'):
df = ta.add_volatility_ta(df, high='high', low='low',
close='close').corr().abs()
df.columns = df.columns.str.replace('volatility_', '')
df.index = df.index.str.replace('volatility_', '')
if (type == 'others'):
df = ta.add_others_ta(df, close='close').corr().abs()
df.columns = df.columns.str.replace('volumeothers_', '')
df.index = df.index.str.replace('volumeothers_', '')
if (type == 'all'):
df = ta.add_all_ta_features(df,
open='open',
high='high',
low='low',
close='close',
volume='volume').corr().abs()
# Remove this from heatmap
df = df.drop(index=['open', 'high', 'low', 'close', 'volume'],
columns=['open', 'high', 'low', 'close', 'volume'])
# Plot the heatmap
plt.subplots(figsize=(10, 10))
# https://seaborn.pydata.org/tutorial/color_palettes.html for possible colors
sns.heatmap(df,
annot=False,
linewidth=0,
center=0.5,
cmap=sns.color_palette("viridis", as_cmap=True))
plt.tight_layout()
plt.yticks(rotation=0)
plt.show()
