def process_one(df): df = utils.dropna(df) if args.custom_indicator_fn is not None: if args.custom_indicator is not None: raise ValueError('custom indicator fn and custom indicator only can use one') # print(args.custom_indicator_fn) df = add_custom_ta(df, args.custom_indicator_fn + '(' + args.custom_indicator_args + ')') elif args.custom_indicator is not None: df = add_custom_ta(df, args.custom_indicator) elif args.indicator == 'all': df = add_all_ta(df, "open", "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix) elif args.indicator == 'trend': df = add_trend_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix) elif args.indicator == 'momentum': df = add_momentum_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix) elif args.indicator == 'my': df = add_my_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix) elif args.indicator == 'volatility': df = add_volatility_ta(df, "high", "low", "close", fillna=True, colprefix=args.column_prefix) elif args.indicator == 'volume': df = add_volume_ta(df, "high", "low", "close", 'volume', fillna=True, colprefix=args.column_prefix) else: raise ValueError('Please use a calculation') return df
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Add all ta features # Clean NaN values dataframe = dropna(dataframe) dataframe['trend_ichimoku_base'] = ta.trend.ichimoku_base_line( dataframe['high'], dataframe['low'], window1=9, window2=26, visual=False, fillna=False) KST = ta.trend.KSTIndicator(close=dataframe['close'], roc1=10, roc2=15, roc3=20, roc4=30, window1=10, window2=10, window3=10, window4=15, nsig=9, fillna=False) dataframe['trend_kst_diff'] = KST.kst_diff() # Normalization tib = dataframe['trend_ichimoku_base'] dataframe['trend_ichimoku_base'] = (tib - tib.min()) / (tib.max() - tib.min()) tkd = dataframe['trend_kst_diff'] dataframe['trend_kst_diff'] = (tkd - tkd.min()) / (tkd.max() - tkd.min()) return dataframe
def generate(self): df = pd.read_csv(self.source_path / 'close.csv', parse_dates=['date'], date_parser=from_small_date) df = dropna(df) close = df['close'] self._macd(df, close) self._bollinger_bands(df, close) self._rsi(df, close) return df
def add_ta(self, pair, interval): self.letgo = True # if(pair == "DGBUSDT"): # self.letgo = True if (self.letgo): #fix 1inch error = True if pair == '1INCHUSDT': pair = 'A1INCHUSDT' # check if table already exists ?????? dfch = pandas.DataFrame() try: conn = sqlite3.connect(self.database) c = conn.cursor() c.execute("SELECT * FROM " + pair + "_" + interval + "_TA") rows = c.fetchall() dfch = pandas.DataFrame(rows) error = False except Exception as e: error = True if (len(dfch) > 1) and not error: # donothing print("doing nothing about : " + pair) # pass else: try: print("going for " + pair) conn = sqlite3.connect(self.database) c = conn.cursor() c.execute( "SELECT timespan1, CAST(open as float) as open ,CAST(high as float) as high ,CAST(low as float) as low ,CAST(close as float) as close,CAST(volume as float) as volume FROM " + pair + "_" + interval + " order by timespan1") rows = c.fetchall() df = pandas.DataFrame(rows) df.columns = [desc[0] for desc in c.description] # Clean nan values df = utils.dropna(df) # print(df.columns) # Add all ta features filling nans values df = ta.add_all_ta_features(df, "open", "high", "low", "close", "volume", fillna=True) # print(df.columns) # print(len(df.columns)) # print(df) except Exception as e: print( 'Failed to add technical analysis lab to database because of : ' + str(e)) self.save_data_with_ta(df, pair, interval)
def add_features(e): e.data = dropna(e.data) e.data = add_all_ta_features(e.data, open="Open", high="High", low="Low", close="Close", volume="Volume") # 43 for trend_trix e.data = e.data[43:] return e
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Add all ta features dataframe = dropna(dataframe) dataframe = add_all_ta_features(dataframe, open="open", high="high", low="low", close="close", volume="volume", fillna=True) # dataframe.to_csv("df.csv", index=True) return dataframe
def get_CSCO_PX_REL_VWAP_data(): df = pd.read_csv('data/input CSCO PX_REL_VWAP.txt', delimiter='\t', index_col=0) df.rename(columns={'Cisco Systems Inc / SPDR S&P 500 ETF Trust': 'close'}, inplace=True) df = utils.dropna(df) df = df.reindex(index=df.index[::-1]) df['open'] = df['close'] df['high'] = df['close'] df['low'] = df['close'] df['volume'] = df['close'] df = add_all_ta(df, open='open', high='high', low='low', close='close', volume='volume', fillna=True, colprefix=args.column_prefix) # df=add_custom_ta(df, 'all.linear_regression(close,n=14)') df.fillna(0, inplace=True) return df
def __init__(self, period, interval): self.time = np.random.randint(low=0, high=MAX_INDEX - WINDOW) self.tickers = [] states = fetch_data(period, interval) for state in states: state_edit = state.drop(columns=['Dividends', 'Stock Splits']) state_edit = dropna(state_edit) state_edit = state_edit.reset_index() # state_edit = add_all_ta_features( # state_edit, open="Open", high="High", low="Low", close="Close", volume="Volume", fillna=True) self.tickers.append(state_edit) # state.iloc[START_INDEX:START_INDEX+WINDOW][["High", "Low", "Close", "Volume"]].to_numpy()) self.tick()
def pull_data(self): with open(f'data/{self.symbol}_{self.timeframe}', newline='') as csvfile: spamreader = csv.reader(csvfile, delimiter=';', quotechar='|') for row in spamreader: #print(row) candle = { "open": float(row[0]), "close": float(row[1]), "high": float(row[2]), "low": float(row[3]), "time": row[4] } self.all_candles.append(candle) self.df = pd.DataFrame(self.all_candles) self.df = dropna(self.df) self.add_features()
def get_single_sequence(df): df = dropna(df) df = add_all_ta_features(df, open="Open", high="High", low="Low", close="Close", volume="Volume") df = df[50:] df = df[FEATURES] df = df.to_numpy() df = (df - df.min(axis=0)) / (df.max(axis=0) - df.min(axis=0)) df = df[-HISTORY_SIZE:] df = df.reshape(1, df.shape[0], df.shape[1]) print(FEATURES) print(df.squeeze()) return df
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe = dropna(dataframe) dataframe['volatility_kcw'] = ta.volatility.keltner_channel_wband( dataframe['high'], dataframe['low'], dataframe['close'], window=20, window_atr=10, fillna=False, original_version=True) dataframe['volatility_dcp'] = ta.volatility.donchian_channel_pband( dataframe['high'], dataframe['low'], dataframe['close'], window=10, offset=0, fillna=False) return dataframe
import pandas as pd from ta.utils import dropna import matplotlib.pyplot as plt from ta.trend import MACD import numpy as np df = pd.read_csv('TSLA.csv', sep=',') # clean NaN df = dropna(df) macd = MACD(close=df["Close"]) def normalize(col): return (col - col.mean()) / col.std() df["macd"] = normalize(macd.macd()) df["macd_signal"] = normalize(macd.macd_signal()) df["macd_diff"] = normalize(macd.macd_diff()) plt.plot(df.macd, label="MACD") plt.plot(df.macd_signal, label="MACD signal") plt.plot(df.macd_diff, label="MACD diff") plt.title("TSLA MACD") plt.show() with open("TSLA_ta.csv", "w") as f: f.write(df.to_csv())
from ray.tune.registry import register_env import ray import ray.rllib.agents.ppo as ppo # from stable_baselines.common.policies import MlpLnLstmPolicy # from stable_baselines import PPO2, A2C import os from ta import add_all_ta_features from ta.utils import dropna import numpy as np import yfinance as yf cdd = CryptoDataDownload() # feature engineer data = cdd.fetch("Bitstamp", "USD", "BTC", "minute") # data = yf.download("EURUSD=X", start="2021-01-01", end="2021-01-31", interval='15m') data = dropna(data) # print(data) data = add_all_ta_features(data, open="open", high="high", low="low", close="close", volume="volume") data = data.dropna(1) print(data) # exit() def rsi(price: Stream[float], period: float) -> Stream[float]: r = price.diff() upside = r.clamp_min(0).abs()
import pandas as pd import matplotlib.pyplot as plt from ta import add_all_ta_features from ta.utils import dropna from ta.volatility import BollingerBands from datetime import datetime import plotly go = plotly.graph_objs data = pd.read_csv("data/Binance_BTCUSDT_1h.csv").head(100) data = data.iloc[::-1] btc_data = dropna(data) # Define the parameters for the Bollinger Band calculation ma_size = 20 bol_size = 2 # Convert the timestamp data to a human readable format btc_data.index = btc_data['Date'] # Calculate the SMA btc_data.insert(0, 'moving_average', btc_data['Close'].rolling(ma_size).mean()) # Calculate the upper and lower Bollinger Bands btc_data.insert(0, 'bol_upper', btc_data['moving_average'] + btc_data['Close'].rolling(ma_size).std() * bol_size) btc_data.insert(0, 'bol_lower', btc_data['moving_average'] - btc_data['Close'].rolling(ma_size).std() * bol_size) btc_data.insert(0, 'Action', 0)
def SuperTrend(data, l=3, h=7): data = ST(data, l, h) data = dropna(data) STSell = (data['SuperTrend'] > data['4. close']) STBuy = (data['SuperTrend'] < data['4. close']) return STSell, STBuy, data
def read_csv(csv_path): data = pd.read_csv(csv_path) data = dropna(data) data = pd.DataFrame(data.values[::-1], data.index, data.columns) return data
Stock_Failure += 1 print("The amount of stocks we successfully imported: " + str(i - Stocks_Not_Imported)) # These two lines remove the Stocks folder and then recreate it in order to remove old stocks. Make sure you have created a Stocks Folder the first time you run this. shutil.rmtree("<Your Path>\\Bayesian_Logistic_Regression\\Stocks_Sub\\") os.mkdir("<Your Path>\\Bayesian_Logistic_Regression\\Stocks_Sub\\") # Get the Y values list_files = ( glob.glob("<Your Path>\\Bayesian_Logistic_Regression\\Stocks\\*.csv") ) # Creates a list of all csv filenames in the stocks folder for interval in list_files: Stock_Name = ((os.path.basename(interval)).split(".csv")[0]) data = pd.read_csv(interval) dropna(data) data = add_all_ta_features(data, open="Open", high="High", low="Low", close="Close", volume="Volume") data = data.iloc[100:] close_prices = data['Close'].tolist() Five_Day_Obs = [] thirty_Day_Obs = [] sixty_Day_Obs = [] x = 0 while x < (len(data)): if x < (len(data) - 5): if ((close_prices[x + 1] + close_prices[x + 2] +
# request historical candle (or klines) data # bars = client.get_historical_klines(pair, tf, '{} days ago UTC'.format((period + 3) // 2)) # bars = client.get_historical_klines(pair, tf, timestamp, 1000) bars = get_api_klines(pair, tf) df_bars = pd.DataFrame( bars, columns=[ 'Open time', 'Open', 'High', 'Low', 'Close', 'Volume', 'Close time', 'Quote asset volume', 'Number of trades', 'Taker buy base asset volume', 'Taker buy quote asset volume', 'Ignore' ]) df_bars = dropna(df_bars) df_bars['Open'] = pd.to_numeric(df_bars['Open'], errors='coerce') df_bars['Close'] = pd.to_numeric(df_bars['Close'], errors='coerce') df_bars['High'] = pd.to_numeric(df_bars['High'], errors='coerce') df_bars['Low'] = pd.to_numeric(df_bars['Low'], errors='coerce') df_bars['Quote asset volume'] = pd.to_numeric( df_bars['Quote asset volume'], errors='coerce') # Initialize Bollinger Bands Indicator indicator_bb = BollingerBands(close=df_bars["Close"], window=20, window_dev=2)
dao = StocksDao() equity_ref_df = dao.get_all_stocks(exchange) eod_df = dao.get_all_stocks_prices(exchange) # Calac data last_entry_df = dao.get_all_stocks_prices_max_entry_date(exchange) last_entry_dict = dict(zip(last_entry_df['equity_id'].tolist(), last_entry_df['last_entry'].tolist())) for idx1, row1 in equity_ref_df.iterrows(): equity_id = row1['equity_id'] ticker = row1['local_code'] try: insert_list = list() # get all the prices for equity ticker_eod_df = eod_df[eod_df['equity_id'] == equity_id] ticker_eod_df = ticker_eod_df.sort_values(by=['trading_date']) df = dropna(ticker_eod_df) # Initialize Bollinger Bands Indicator indicator_bb = BollingerBands(close=df["adj_close"], window=20, window_dev=2) # Add Bollinger Bands features df['bb_bbm'] = indicator_bb.bollinger_mavg() df['bb_bbh'] = indicator_bb.bollinger_hband() df['bb_bbl'] = indicator_bb.bollinger_lband() # EMA Indicator indicator_ema_200 = EMAIndicator(close=df["adj_close"], window=200) df['ema_200'] = indicator_ema_200.ema_indicator() indicator_ema_100 = EMAIndicator(close=df["adj_close"], window=100) df['ema_100'] = indicator_ema_100.ema_indicator() indicator_ema_50 = EMAIndicator(close=df["adj_close"], window=50) df['ema_50'] = indicator_ema_50.ema_indicator()
def GetPriceDataFromExchangeFinnHub(event, context): retVal = {} retVal["data"] = [] # For debugging the input, write the EVENT to CloudWatch logs print(json.dumps(event)) # Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake event_body = event["body"] payload = json.loads(event_body) for row in payload["data"]: sflkRowRef = row[ 0] # This is how Snowflake keeps track of data as it gets returned symbol = row[ 1] # The data passed in from Snowflake that the input row contains. fromDate = row[2] toDate = row[3] # Will return URL without token to Snowflake for tracking URL = f'https://finnhub.io/api/v1/stock/candle?symbol={symbol}&resolution=D&from={fromDate}&to={toDate}' # Add our FinnHubAPI Key to the end of the URL. # This is in a new variable which will not be returned to Snowflake URLWithToken = f'{URL}&token={FinnHubAPI.TOKEN}' # GET data from the API httpData = requests.get(url=URLWithToken).json() # Convert to Pandas DataFrame df = pd.DataFrame(httpData) # Add the column names print("Adding column names") df.columns = [ "Close", "High", "Low", "Open", "Status", "OpenTime", "Volume" ] # Set DateTime columns to correct type df['OpenTime'] = pd.to_datetime(df['OpenTime'], unit='ms') df['Open'] = df['Open'].astype(float) df['High'] = df['High'].astype(float) df['Low'] = df['Low'].astype(float) df['Close'] = df['Close'].astype(float) df['Volume'] = df['Volume'].astype(float) # Clean NaN values print("Cleaning NA values") df = dropna(df) # Calculate the Bollinger Bands indicator indicator_bb = BollingerBands(close=df["Close"], n=20, ndev=2) df['bb_bbm'] = indicator_bb.bollinger_mavg() df['bb_bbh'] = indicator_bb.bollinger_hband() df['bb_bbl'] = indicator_bb.bollinger_lband() df['bb_bbhi'] = indicator_bb.bollinger_hband_indicator() df['bb_bbli'] = indicator_bb.bollinger_lband_indicator() df['bb_bbw'] = indicator_bb.bollinger_wband() df['bb_bbp'] = indicator_bb.bollinger_pband() print("converting OHLC pandas to JSON. This does it as a string") buffer = df.to_json(orient="records") print("Interpret the JSON string into a dictionary for output") jsonResponse = json.loads(buffer) # Prepare the output response response = {} response["url"] = URL response["response"] = jsonResponse retVal["data"].append([sflkRowRef, response]) # For debugging the output, write the RETurn VALue to CloudWatch logs # print(json.dumps(retVal)) return retVal