def process_one(df):
df = utils.dropna(df)
if args.custom_indicator_fn is not None:
if args.custom_indicator is not None:
raise ValueError('custom indicator fn and custom indicator only can use one')
# print(args.custom_indicator_fn)
df = add_custom_ta(df, args.custom_indicator_fn + '(' + args.custom_indicator_args + ')')
elif args.custom_indicator is not None:
df = add_custom_ta(df, args.custom_indicator)
elif args.indicator == 'all':
df = add_all_ta(df, "open", "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'trend':
df = add_trend_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'momentum':
df = add_momentum_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'my':
df = add_my_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'volatility':
df = add_volatility_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix)
elif args.indicator == 'volume':
df = add_volume_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix)
else:
raise ValueError('Please use a calculation')
return df
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
# Add all ta features
# Clean NaN values
dataframe = dropna(dataframe)
dataframe['trend_ichimoku_base'] = ta.trend.ichimoku_base_line(
dataframe['high'],
dataframe['low'],
window1=9,
window2=26,
visual=False,
fillna=False)
KST = ta.trend.KSTIndicator(close=dataframe['close'],
roc1=10,
roc2=15,
roc3=20,
roc4=30,
window1=10,
window2=10,
window3=10,
window4=15,
nsig=9,
fillna=False)
dataframe['trend_kst_diff'] = KST.kst_diff()
# Normalization
tib = dataframe['trend_ichimoku_base']
dataframe['trend_ichimoku_base'] = (tib - tib.min()) / (tib.max() -
tib.min())
tkd = dataframe['trend_kst_diff']
dataframe['trend_kst_diff'] = (tkd - tkd.min()) / (tkd.max() -
tkd.min())
return dataframe
def generate(self):
df = pd.read_csv(self.source_path / 'close.csv', parse_dates=['date'], date_parser=from_small_date)
df = dropna(df)
close = df['close']
self._macd(df, close)
self._bollinger_bands(df, close)
self._rsi(df, close)
return df
def add_ta(self, pair, interval):
self.letgo = True
# if(pair == "DGBUSDT"):
# self.letgo = True
if (self.letgo):
#fix 1inch
error = True
if pair == '1INCHUSDT':
pair = 'A1INCHUSDT'
# check if table already exists ??????
dfch = pandas.DataFrame()
try:
conn = sqlite3.connect(self.database)
c = conn.cursor()
c.execute("SELECT * FROM " + pair + "_" + interval + "_TA")
rows = c.fetchall()
dfch = pandas.DataFrame(rows)
error = False
except Exception as e:
error = True
if (len(dfch) > 1) and not error:
# donothing
print("doing nothing about : " + pair)
# pass
else:
try:
print("going for " + pair)
conn = sqlite3.connect(self.database)
c = conn.cursor()
c.execute(
"SELECT timespan1, CAST(open as float) as open ,CAST(high as float) as high ,CAST(low as float) as low ,CAST(close as float) as close,CAST(volume as float) as volume FROM "
+ pair + "_" + interval + " order by timespan1")
rows = c.fetchall()
df = pandas.DataFrame(rows)
df.columns = [desc[0] for desc in c.description]
# Clean nan values
df = utils.dropna(df)
# print(df.columns)
# Add all ta features filling nans values
df = ta.add_all_ta_features(df,
"open",
"high",
"low",
"close",
"volume",
fillna=True)
# print(df.columns)
# print(len(df.columns))
# print(df)
except Exception as e:
print(
'Failed to add technical analysis lab to database because of : '
+ str(e))
self.save_data_with_ta(df, pair, interval)
def add_features(e):
e.data = dropna(e.data)
e.data = add_all_ta_features(e.data,
open="Open",
high="High",
low="Low",
close="Close",
volume="Volume")
# 43 for trend_trix
e.data = e.data[43:]
return e
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
# Add all ta features
dataframe = dropna(dataframe)
dataframe = add_all_ta_features(dataframe,
open="open",
high="high",
low="low",
close="close",
volume="volume",
fillna=True)
# dataframe.to_csv("df.csv", index=True)
return dataframe
def get_CSCO_PX_REL_VWAP_data():
df = pd.read_csv('data/input CSCO PX_REL_VWAP.txt', delimiter='\t', index_col=0)
df.rename(columns={'Cisco Systems Inc / SPDR S&P 500 ETF Trust': 'close'}, inplace=True)
df = utils.dropna(df)
df = df.reindex(index=df.index[::-1])
df['open'] = df['close']
df['high'] = df['close']
df['low'] = df['close']
df['volume'] = df['close']
df = add_all_ta(df, open='open', high='high', low='low', close='close', volume='volume', fillna=True,
colprefix=args.column_prefix)
# df=add_custom_ta(df, 'all.linear_regression(close,n=14)')
df.fillna(0, inplace=True)
return df
def __init__(self, period, interval):
self.time = np.random.randint(low=0, high=MAX_INDEX - WINDOW)
self.tickers = []
states = fetch_data(period, interval)
for state in states:
state_edit = state.drop(columns=['Dividends', 'Stock Splits'])
state_edit = dropna(state_edit)
state_edit = state_edit.reset_index()
# state_edit = add_all_ta_features(
# state_edit, open="Open", high="High", low="Low", close="Close", volume="Volume", fillna=True)
self.tickers.append(state_edit)
# state.iloc[START_INDEX:START_INDEX+WINDOW][["High", "Low", "Close", "Volume"]].to_numpy())
self.tick()
def pull_data(self):
with open(f'data/{self.symbol}_{self.timeframe}',
newline='') as csvfile:
spamreader = csv.reader(csvfile, delimiter=';', quotechar='|')
for row in spamreader:
#print(row)
candle = {
"open": float(row[0]),
"close": float(row[1]),
"high": float(row[2]),
"low": float(row[3]),
"time": row[4]
}
self.all_candles.append(candle)
self.df = pd.DataFrame(self.all_candles)
self.df = dropna(self.df)
self.add_features()
def get_single_sequence(df):
df = dropna(df)
df = add_all_ta_features(df,
open="Open",
high="High",
low="Low",
close="Close",
volume="Volume")
df = df[50:]
df = df[FEATURES]
df = df.to_numpy()
df = (df - df.min(axis=0)) / (df.max(axis=0) - df.min(axis=0))
df = df[-HISTORY_SIZE:]
df = df.reshape(1, df.shape[0], df.shape[1])
print(FEATURES)
print(df.squeeze())
return df
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
dataframe = dropna(dataframe)
dataframe['volatility_kcw'] = ta.volatility.keltner_channel_wband(
dataframe['high'],
dataframe['low'],
dataframe['close'],
window=20,
window_atr=10,
fillna=False,
original_version=True)
dataframe['volatility_dcp'] = ta.volatility.donchian_channel_pband(
dataframe['high'],
dataframe['low'],
dataframe['close'],
window=10,
offset=0,
fillna=False)
return dataframe
import pandas as pd
from ta.utils import dropna
import matplotlib.pyplot as plt
from ta.trend import MACD
import numpy as np
df = pd.read_csv('TSLA.csv', sep=',')
# clean NaN
df = dropna(df)
macd = MACD(close=df["Close"])
def normalize(col):
return (col - col.mean()) / col.std()
df["macd"] = normalize(macd.macd())
df["macd_signal"] = normalize(macd.macd_signal())
df["macd_diff"] = normalize(macd.macd_diff())
plt.plot(df.macd, label="MACD")
plt.plot(df.macd_signal, label="MACD signal")
plt.plot(df.macd_diff, label="MACD diff")
plt.title("TSLA MACD")
plt.show()
with open("TSLA_ta.csv", "w") as f:
f.write(df.to_csv())
from ray.tune.registry import register_env
import ray
import ray.rllib.agents.ppo as ppo
# from stable_baselines.common.policies import MlpLnLstmPolicy
# from stable_baselines import PPO2, A2C
import os
from ta import add_all_ta_features
from ta.utils import dropna
import numpy as np
import yfinance as yf
cdd = CryptoDataDownload()
# feature engineer
data = cdd.fetch("Bitstamp", "USD", "BTC", "minute")
# data = yf.download("EURUSD=X", start="2021-01-01", end="2021-01-31", interval='15m')
data = dropna(data)
# print(data)
data = add_all_ta_features(data,
open="open",
high="high",
low="low",
close="close",
volume="volume")
data = data.dropna(1)
print(data)
# exit()
def rsi(price: Stream[float], period: float) -> Stream[float]:
r = price.diff()
upside = r.clamp_min(0).abs()
import pandas as pd
import matplotlib.pyplot as plt
from ta import add_all_ta_features
from ta.utils import dropna
from ta.volatility import BollingerBands
from datetime import datetime
import plotly
go = plotly.graph_objs
data = pd.read_csv("data/Binance_BTCUSDT_1h.csv").head(100)
data = data.iloc[::-1]
btc_data = dropna(data)
# Define the parameters for the Bollinger Band calculation
ma_size = 20
bol_size = 2
# Convert the timestamp data to a human readable format
btc_data.index = btc_data['Date']
# Calculate the SMA
btc_data.insert(0, 'moving_average', btc_data['Close'].rolling(ma_size).mean())
# Calculate the upper and lower Bollinger Bands
btc_data.insert(0, 'bol_upper', btc_data['moving_average'] + btc_data['Close'].rolling(ma_size).std() * bol_size)
btc_data.insert(0, 'bol_lower', btc_data['moving_average'] - btc_data['Close'].rolling(ma_size).std() * bol_size)
btc_data.insert(0, 'Action', 0)
def SuperTrend(data, l=3, h=7):
data = ST(data, l, h)
data = dropna(data)
STSell = (data['SuperTrend'] > data['4. close'])
STBuy = (data['SuperTrend'] < data['4. close'])
return STSell, STBuy, data
def read_csv(csv_path):
data = pd.read_csv(csv_path)
data = dropna(data)
data = pd.DataFrame(data.values[::-1], data.index, data.columns)
return data
Stock_Failure += 1
print("The amount of stocks we successfully imported: " +
str(i - Stocks_Not_Imported))
# These two lines remove the Stocks folder and then recreate it in order to remove old stocks. Make sure you have created a Stocks Folder the first time you run this.
shutil.rmtree("<Your Path>\\Bayesian_Logistic_Regression\\Stocks_Sub\\")
os.mkdir("<Your Path>\\Bayesian_Logistic_Regression\\Stocks_Sub\\")
# Get the Y values
list_files = (
glob.glob("<Your Path>\\Bayesian_Logistic_Regression\\Stocks\\*.csv")
) # Creates a list of all csv filenames in the stocks folder
for interval in list_files:
Stock_Name = ((os.path.basename(interval)).split(".csv")[0])
data = pd.read_csv(interval)
dropna(data)
data = add_all_ta_features(data,
open="Open",
high="High",
low="Low",
close="Close",
volume="Volume")
data = data.iloc[100:]
close_prices = data['Close'].tolist()
Five_Day_Obs = []
thirty_Day_Obs = []
sixty_Day_Obs = []
x = 0
while x < (len(data)):
if x < (len(data) - 5):
if ((close_prices[x + 1] + close_prices[x + 2] +
# request historical candle (or klines) data
# bars = client.get_historical_klines(pair, tf, '{} days ago UTC'.format((period + 3) // 2))
# bars = client.get_historical_klines(pair, tf, timestamp, 1000)
bars = get_api_klines(pair, tf)
df_bars = pd.DataFrame(
bars,
columns=[
'Open time', 'Open', 'High', 'Low', 'Close', 'Volume',
'Close time', 'Quote asset volume', 'Number of trades',
'Taker buy base asset volume',
'Taker buy quote asset volume', 'Ignore'
])
df_bars = dropna(df_bars)
df_bars['Open'] = pd.to_numeric(df_bars['Open'],
errors='coerce')
df_bars['Close'] = pd.to_numeric(df_bars['Close'],
errors='coerce')
df_bars['High'] = pd.to_numeric(df_bars['High'],
errors='coerce')
df_bars['Low'] = pd.to_numeric(df_bars['Low'], errors='coerce')
df_bars['Quote asset volume'] = pd.to_numeric(
df_bars['Quote asset volume'], errors='coerce')
# Initialize Bollinger Bands Indicator
indicator_bb = BollingerBands(close=df_bars["Close"],
window=20,
window_dev=2)
dao = StocksDao()
equity_ref_df = dao.get_all_stocks(exchange)
eod_df = dao.get_all_stocks_prices(exchange)
# Calac data
last_entry_df = dao.get_all_stocks_prices_max_entry_date(exchange)
last_entry_dict = dict(zip(last_entry_df['equity_id'].tolist(), last_entry_df['last_entry'].tolist()))
for idx1, row1 in equity_ref_df.iterrows():
equity_id = row1['equity_id']
ticker = row1['local_code']
try:
insert_list = list()
# get all the prices for equity
ticker_eod_df = eod_df[eod_df['equity_id'] == equity_id]
ticker_eod_df = ticker_eod_df.sort_values(by=['trading_date'])
df = dropna(ticker_eod_df)
# Initialize Bollinger Bands Indicator
indicator_bb = BollingerBands(close=df["adj_close"], window=20, window_dev=2)
# Add Bollinger Bands features
df['bb_bbm'] = indicator_bb.bollinger_mavg()
df['bb_bbh'] = indicator_bb.bollinger_hband()
df['bb_bbl'] = indicator_bb.bollinger_lband()
# EMA Indicator
indicator_ema_200 = EMAIndicator(close=df["adj_close"], window=200)
df['ema_200'] = indicator_ema_200.ema_indicator()
indicator_ema_100 = EMAIndicator(close=df["adj_close"], window=100)
df['ema_100'] = indicator_ema_100.ema_indicator()
indicator_ema_50 = EMAIndicator(close=df["adj_close"], window=50)
df['ema_50'] = indicator_ema_50.ema_indicator()
def GetPriceDataFromExchangeFinnHub(event, context):
retVal = {}
retVal["data"] = []
# For debugging the input, write the EVENT to CloudWatch logs
print(json.dumps(event))
# Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake
event_body = event["body"]
payload = json.loads(event_body)
for row in payload["data"]:
sflkRowRef = row[
0] # This is how Snowflake keeps track of data as it gets returned
symbol = row[
1] # The data passed in from Snowflake that the input row contains.
fromDate = row[2]
toDate = row[3]
# Will return URL without token to Snowflake for tracking
URL = f'https://finnhub.io/api/v1/stock/candle?symbol={symbol}&resolution=D&from={fromDate}&to={toDate}'
# Add our FinnHubAPI Key to the end of the URL.
# This is in a new variable which will not be returned to Snowflake
URLWithToken = f'{URL}&token={FinnHubAPI.TOKEN}'
# GET data from the API
httpData = requests.get(url=URLWithToken).json()
# Convert to Pandas DataFrame
df = pd.DataFrame(httpData)
# Add the column names
print("Adding column names")
df.columns = [
"Close", "High", "Low", "Open", "Status", "OpenTime", "Volume"
]
# Set DateTime columns to correct type
df['OpenTime'] = pd.to_datetime(df['OpenTime'], unit='ms')
df['Open'] = df['Open'].astype(float)
df['High'] = df['High'].astype(float)
df['Low'] = df['Low'].astype(float)
df['Close'] = df['Close'].astype(float)
df['Volume'] = df['Volume'].astype(float)
# Clean NaN values
print("Cleaning NA values")
df = dropna(df)
# Calculate the Bollinger Bands indicator
indicator_bb = BollingerBands(close=df["Close"], n=20, ndev=2)
df['bb_bbm'] = indicator_bb.bollinger_mavg()
df['bb_bbh'] = indicator_bb.bollinger_hband()
df['bb_bbl'] = indicator_bb.bollinger_lband()
df['bb_bbhi'] = indicator_bb.bollinger_hband_indicator()
df['bb_bbli'] = indicator_bb.bollinger_lband_indicator()
df['bb_bbw'] = indicator_bb.bollinger_wband()
df['bb_bbp'] = indicator_bb.bollinger_pband()
print("converting OHLC pandas to JSON. This does it as a string")
buffer = df.to_json(orient="records")
print("Interpret the JSON string into a dictionary for output")
jsonResponse = json.loads(buffer)
# Prepare the output response
response = {}
response["url"] = URL
response["response"] = jsonResponse
retVal["data"].append([sflkRowRef, response])
# For debugging the output, write the RETurn VALue to CloudWatch logs
# print(json.dumps(retVal))
return retVal