def anchored_vwap( ohlc_data: df, start_time: datetime, debug=False ) -> pd.Series: try: start_time_index = ohlc_data["close"].index.get_loc( start_time, method="nearest" ) except Exception as e: if debug: tlog(f"IndexError exception {e} in anchored_vwap for {ohlc_data}") return pd.Series() df = ohlc_data.copy() df["pv"] = df.apply( lambda x: (x["close"] + x["high"] + x["low"]) / 3 * x["volume"], axis=1 ) df["apv"] = df["pv"][start_time_index:].cumsum() df["av"] = df["volume"][start_time_index:].cumsum() df["average"] = df["apv"] / df["av"] if debug: tlog( f"\n{tabulate(df.average[start_time_index:][-15:], headers='keys', tablefmt='psql')}" ) tlog( f"\n{tabulate(df.average[start_time_index:][:15], headers='keys', tablefmt='psql')}" ) return df.average[start_time_index:]
def get_historical_data_from_polygon( api: tradeapi, symbols: List[str], max_tickers: int ) -> Dict[str, df]: """get ticker history""" tlog(f"Loading max {max_tickers} tickers w/ highest volume from Polygon") minute_history: Dict[str, df] = {} c = 0 exclude_symbols = [] try: for symbol in symbols: if symbol not in minute_history: retry_counter = 5 while retry_counter > 0: try: if c < max_tickers: _df = api.polygon.historic_agg_v2( symbol, 1, "minute", _from=str(date.today() - timedelta(days=10)), to=str(date.today() + timedelta(days=1)), ).df _df["vwap"] = 0.0 _df["average"] = 0.0 minute_history[symbol] = _df tlog( f"loaded {len(minute_history[symbol].index)} agg data points for {symbol} {c+1}/{max_tickers}" ) c += 1 break exclude_symbols.append(symbol) break except ( requests.exceptions.HTTPError, requests.exceptions.ConnectionError, ): retry_counter -= 1 if retry_counter == 0: exclude_symbols.append(symbol) except KeyboardInterrupt: tlog("KeyboardInterrupt") for x in exclude_symbols: symbols.remove(x) tlog(f"Total number of symbols for trading {len(symbols)}") return minute_history
def consumer_main( queue: Queue, symbols: List[str], minute_history: Dict[str, df], unique_id: str, conf: Dict, ) -> None: tlog(f"*** consumer_main() starting w pid {os.getpid()} ***") try: config.build_label = pygit2.Repository("../").describe( describe_strategy=pygit2.GIT_DESCRIBE_TAGS) except pygit2.GitError: import liualgotrader config.build_label = liualgotrader.__version__ if hasattr( liualgotrader, "__version__") else "" # type: ignore config.bypass_market_schedule = conf.get("bypass_market_schedule", False) market_data.minute_history = minute_history try: if not asyncio.get_event_loop().is_closed(): asyncio.get_event_loop().close() asyncio.run( consumer_async_main(queue, symbols, unique_id, conf["strategies"])) # loop = asyncio.new_event_loop() # asyncio.set_event_loop(asyncio.new_event_loop()) # loop.run_until_complete(consumer_async_main(queue, symbols, unique_id)) # loop.run_forever() except KeyboardInterrupt: tlog("consumer_main() - Caught KeyboardInterrupt") tlog("*** consumer_main() completed ***")
async def minutes_handler(cls, symbol: str, data: Dict, queue: Queue) -> None: if data["ev"] != "AM": tlog( f"AlpacaStreaming.minutes_handler() got invalid event data: {symbol}:{data}" ) return if symbol[3:] != data["T"]: tlog( f"AlpacaStreaming.minutes_handler() symbol does not match data payload {symbol}:{data}" ) return try: data["EV"] = "AM" data["open"] = data["o"] data["high"] = data["h"] data["low"] = data["l"] data["close"] = data["c"] data["volume"] = data["v"] data["vwap"] = data["vw"] data["average"] = data["a"] queue.put(json.dumps(data)) except Exception as e: tlog( f"Exception in handle_minute_bar(): exception of type {type(e).__name__} with args {e.args}" )
async def handle_data_queue_msg(data: Dict, trader: Trader, data_loader: DataLoader) -> bool: global shortable global symbol_data_error global rejects symbol = data["symbol"] shortable[symbol] = True # ToDO if data["EV"] == "T": return await handle_transaction(symbol, data, trader, data_loader) elif data["EV"] == "Q": return await handle_quote(data) elif data["EV"] in ("A", "AM"): original_ts = ts = pd.Timestamp(data["start"], tz="America/New_York", unit="ms") ts = ts.replace(second=0, microsecond=0) await aggregate_bar_data(data_loader, data, pd.to_datetime(ts)) if data["EV"] == "A": if (time_diff := datetime.now(tz=timezone("America/New_York")) - original_ts) > timedelta(seconds=10): # type: ignore tlog(f"A$ {symbol} too out of sync w {time_diff}") return False elif (datetime.now(tz=timezone("America/New_York")).replace( second=0, microsecond=0) > ts): return True elif data["EV"] == "AM": return True return await handle_aggregate( trader=trader, data_loader=data_loader, symbol=symbol, ts=original_ts, data=data, )
def add_daily_vwap(minute_data: df, debug=False) -> bool: back_time = ts(config.market_open) try: back_time_index = minute_data["close"].index.get_loc(back_time, method="nearest") except IndexError as e: if debug: tlog( f"IndexError exception {e} in add_daily_vwap for {minute_data}" ) return False minute_data["pv"] = minute_data.apply( lambda x: (x["close"] + x["high"] + x["low"]) / 3 * x["volume"], axis=1) minute_data["apv"] = minute_data["pv"][back_time_index:].cumsum() minute_data["av"] = minute_data["volume"][back_time_index:].cumsum() minute_data["average"] = minute_data["apv"] / minute_data["av"] minute_data["vwap"] = minute_data.apply( lambda x: (x["close"] + x["high"] + x["low"]) / 3, axis=1) # print(f"\n{tabulate(minute_data, headers='keys', tablefmt='psql')}") if debug: tlog( f"\n{tabulate(minute_data[-110:-100], headers='keys', tablefmt='psql')}" ) tlog( f"\n{tabulate(minute_data[-10:], headers='keys', tablefmt='psql')}" ) return True
async def load_data(self, symbols: List[str]) -> None: if not len(symbols): raise Exception( "load_data() received an empty list of symbols to load. aborting" ) for i, symbol in enumerate(symbols, start=1): if self.debug: tlog( f"loading 200 days for symbol {symbol} ({i}/{len(symbols)})" ) try: self.data_bars[symbol] = self.data_loader[symbol][date.today( ) - timedelta(days=int(200 * 7 / 5)):date.today() # type: ignore ] if self.debug: try: p_points = len(self.data_bars[symbol]) except TypeError: p_points = 0 tlog(f"loaded at least {p_points} relevant data-points") except Exception: tlog(f"[ERROR] could not load all data points for {symbol}") self.data_bars[symbol] = None
async def run(self) -> bool: if not self.symbols: self.symbols = await TickerData.load_symbols() if not self.symbols: return False # check last date for symbol in self.symbols: latest_date = await StockOhlc.get_latest_date(symbol) if self._debug: tlog(f"{symbol} latest date: {latest_date}") if not latest_date: if self._debug: tlog(f"{symbol} loading {self.days} of OHLC data") await self.load_symbol_data(symbol, self.days) else: latest_date += timedelta(days=1) duration = min(self.days, (date.today() - latest_date).days) if self._debug: tlog(f"{symbol} loading {duration} of OHLC data") await self.load_symbol_data(symbol, duration) return True
async def find_supports( symbol: str, strategy_name: str, current_value: float, minute_history: df, debug=False, ) -> List[float]: """calculate supports""" for back_track_min in range(120, len(minute_history.index), 60): series = ( minute_history["close"][-back_track_min:].dropna().between_time( "9:30", "16:00").resample("15min").min()).dropna() diff = np.diff(series.values) high_index = np.where((diff[:-1] <= 0) & (diff[1:] > 0))[0] + 1 if len(high_index) > 0: local_maximas = sorted( [series[i] for i in high_index if series[i] <= current_value]) if len(local_maximas) > 0: if debug: tlog("find_supports()") tlog(f"{minute_history}") tlog(f"{minute_history['close'][-1]}, {series}") # tlog( # f"[{strategy_name}] find_supports({symbol})={local_maximas}" # ) return local_maximas return []
async def run_all( self, symbols_position: Dict[str, float], data_loader: DataLoader, now: datetime, portfolio_value: float = None, trader: Trader = None, debug: bool = False, backtesting: bool = False, ) -> Dict[str, Dict]: self.key = f"{self.portfolio_id}-{self.name}-last-rebalance" if await self.should_rebalance(now): tlog("time for rebalance") portfolio_symbols_position = await self.load_symbol_position() portfolio_symbols_position = { symbol.upper(): portfolio_symbols_position[symbol] for symbol in portfolio_symbols_position.keys() } tlog(f"current positions {portfolio_symbols_position}") return await self.rebalance( data_loader, trader, portfolio_symbols_position, now ) else: tlog(f"skip rebalance {now}") return {}
async def apply_filters(self) -> None: d = df(self.portfolio) for c, (i, row) in enumerate(self.portfolio.iterrows(), start=1): indicator_calculator = StockDataFrame(self.data_bars[row.symbol]) removed = False for indicator in self.indicators: if indicator == "SMA100": sma_100 = indicator_calculator["close_100_sma"] if self.debug: tlog( f"indicator {indicator} for {row.symbol} ({c}/{len(self.portfolio)}) : {sma_100[-1]}" ) if self.data_bars[row.symbol].close[-1] < sma_100[-1]: if self.debug: tlog(f"{row.symbol} REMOVED on SMA") d = d.drop(index=i) removed = True # filter stocks moving > 15% in last 90 days high = self.data_bars[row.symbol].close[ -1 ] # self.data_bars[row.symbol].close[-90:].max() low = self.data_bars[row.symbol].close[-90:].min() if not removed and high / low > 1.25 and self.debug: tlog( f"{row.symbol} ({c}/{len(self.portfolio)}) REMOVED on movement ({high},{low})> 25% in last 90 days" ) d = d.drop(index=i) removed = True self.portfolio = d
async def find_resistances( symbol: str, strategy_name: str, current_value: float, minute_history: df, debug=False, ) -> Optional[List[float]]: """calculate supports""" est = pytz.timezone("America/New_York") back_time = ts(datetime.now(est)).to_pydatetime() - timedelta(days=3) back_time_index = minute_history["close"].index.get_loc(back_time, method="nearest") series = (minute_history["close"][back_time_index:].dropna().between_time( "9:30", "16:00").resample("15min").max()).dropna() diff = np.diff(series.values) high_index = np.where((diff[:-1] >= 0) & (diff[1:] <= 0))[0] + 1 if len(high_index) > 0: local_maximas = sorted( [series[i] for i in high_index if series[i] >= current_value]) if len(local_maximas) > 0: if debug: tlog("find_resistances()") tlog(f"{minute_history}") tlog(f"{minute_history['close'][-1]}, {series}") # tlog( # f"[{strategy_name}] find_resistances({symbol})={local_maximas}" # ) return local_maximas return None
async def producer_async_main( queues: List[Queue], scanner_queue: Queue, num_consumer_processes: int, ): await create_db_connection(str(config.dsn)) await run(queues=queues) trade_ws = tradeapi.StreamConn( base_url=config.alpaca_base_url, key_id=config.alpaca_api_key, secret_key=config.alpaca_api_secret, ) trade_updates_task = asyncio.create_task( trade_run(ws=trade_ws, queues=queues), name="trade_updates_task", ) scanner_input_task = asyncio.create_task( scanner_input(scanner_queue, queues, num_consumer_processes), name="scanner_input", ) tear_down = asyncio.create_task( teardown_task( timezone("America/New_York"), [trade_ws], [scanner_input_task], ) ) await asyncio.gather( trade_updates_task, scanner_input_task, tear_down, return_exceptions=True, ) tlog("producer_async_main() completed")
async def handle_trade_update_wo_order(data: Dict) -> bool: symbol = data["symbol"] event = data["event"] tlog( f"trade update without order for {symbol} data={data} with event {event}" ) algo_run_id = await NewTrade.get_latest_algo_run_id(symbol=symbol) tlog(f"found algo_run_id {algo_run_id}") for s in trading_data.strategies: if s.algo_run.run_id == algo_run_id: trading_data.last_used_strategy[symbol] = s tlog(f"found strategy {str(s)}") break if event == "partial_fill" and symbol in trading_data.last_used_strategy: await update_partially_filled_order( trading_data.last_used_strategy[symbol], Order(data["order"])) elif event == "fill" and symbol in trading_data.last_used_strategy: await update_filled_order(trading_data.last_used_strategy[symbol], Order(data["order"])) elif event in ("canceled", "rejected"): trading_data.partial_fills.pop(symbol, None) return True
def consumer_main( queue: Queue, symbols: List[str], unique_id: str, conf: Dict, ) -> None: tlog(f"*** consumer_main() starting w pid {os.getpid()} ***") try: config.build_label = pygit2.Repository("../").describe( describe_strategy=pygit2.GIT_DESCRIBE_TAGS) except pygit2.GitError: import liualgotrader config.build_label = liualgotrader.__version__ if hasattr( liualgotrader, "__version__") else "" # type: ignore config.bypass_market_schedule = conf.get("bypass_market_schedule", False) config.portfolio_value = conf.get("portfolio_value", None) if "risk" in conf: config.risk = conf["risk"] if "market_liquidation_end_time_minutes" in conf: config.market_liquidation_end_time_minutes = conf[ "market_liquidation_end_time_minutes"] try: asyncio.run( consumer_async_main(queue, symbols, unique_id, conf["strategies"])) except KeyboardInterrupt: tlog("consumer_main() - Caught KeyboardInterrupt") tlog("*** consumer_main() completed ***")
def backtest(from_date: date, to_date: date, scale: TimeScale, config: Dict) -> str: uid = str(uuid.uuid4()) try: if not asyncio.get_event_loop().is_closed(): asyncio.get_event_loop().close() loop = asyncio.new_event_loop() asyncio.set_event_loop(asyncio.new_event_loop()) loop.run_until_complete( backtest_main(uid, from_date, to_date, scale, config)) except KeyboardInterrupt: tlog("backtest() - Caught KeyboardInterrupt") except Exception as e: tlog( f"backtest() - exception of type {type(e).__name__} with args {e.args}" ) traceback.print_exc() finally: print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") print(f"new batch-id: {uid}") return uid
async def do_strategy_all( data_loader: DataLoader, now: pd.Timestamp, strategy: Strategy, ): try: do = await strategy.run_all( symbols_position=trading_data.positions, now=now.to_pydatetime(), portfolio_value=portfolio_value, backtesting=True, data_loader=data_loader, ) items = list(do.items()) items.sort(key=lambda x: int(x[1]["side"] == "buy")) for symbol, what in items: await do_strategy_result(strategy, symbol, now, what) except Exception as e: tlog(f"[Exception] {now} {strategy}->{e}") traceback.print_exc() raise
async def create_scanners( data_loader: DataLoader, scanners_conf: Dict, scanner_names: Optional[List], ) -> List[Scanner]: scanners: List = [] for scanner_name in scanners_conf: if scanner_names and scanner_name not in scanner_name: continue tlog(f"scanner {scanner_name} selected") if scanner_name == "momentum": tlog( "momentum scanner can not be supported in backtest on time-frame. skipping" ) else: scanners.append(await Scanner.get_scanner( data_loader=data_loader, scanner_name=scanner_name, scanner_details=scanners_conf[scanner_name], )) return scanners
def _fetch(self, session: requests.Session, page: int) -> List[Ticker]: url = "https://api.polygon.io/" + "v2" + "/reference/tickers" try: with session.get( url, params={ "apiKey": get_polygon_credentials(config.prod_api_key_id), "market": "STOCKS", "page": page, "active": "true", "perpage": 50, }, ) as response: data = response.json()["tickers"] return [Ticker(x) for x in data] except requests.exceptions.ConnectionError as e: tlog( f"_fetch(): got HTTP exception {e}, for {page}, going to sleep, then retry" ) time.sleep(30) return self._fetch(requests.Session(), page)
def __init__( self, data: Dict, debug=False, ): try: self.rank_days = int(data["rank_days"]) self.atr_days = int(data["atr_days"]) self.index = data["index"] self.debug = debug self.portfolio_size = data["portfolio_size"] self.risk_factor = data["risk_factor"] self.data_loader = DataLoader(TimeScale.day) except Exception: raise ValueError( "[ERROR] Miner must receive all valid parameter(s)" ) super().__init__(name="PortfolioBuilder") if self.debug: tlog(f"{self.name} running in debug mode")
async def liquidate( symbol: str, symbol_position: int, trading_api: tradeapi, ) -> None: if symbol_position and symbol not in trading_data.open_orders: tlog( f"Trading over, trying to liquidate remaining position {symbol_position} in {symbol}" ) try: if symbol_position < 0: o = trading_api.submit_order( symbol=symbol, qty=str(-symbol_position), side="buy", type="market", time_in_force="day", ) op = "buy" trading_data.buy_indicators[symbol] = {"liquidation": 1} else: o = trading_api.submit_order( symbol=symbol, qty=str(symbol_position), side="sell", type="market", time_in_force="day", ) op = "sell" trading_data.sell_indicators[symbol] = {"liquidation": 1} trading_data.open_orders[symbol] = (o, op) trading_data.open_order_strategy[ symbol] = trading_data.last_used_strategy[symbol] except Exception as e: tlog(f"failed to liquidate {symbol} w exception {e}")
async def aload_trades_by_portfolio_id(portfolio_id: str) -> pd.DataFrame: query = f""" SELECT t.*, a.batch_id, a.start_time, a.algo_name FROM new_trades as t, algo_run as a, portfolio_batch_ids as p WHERE t.algo_run_id = a.algo_run_id AND a.batch_id = p.batch_id AND p.portfolio_id = '{portfolio_id}' AND t.expire_tstamp is null ORDER BY symbol, tstamp """ df: pd.DataFrame = await fetch_as_dataframe(query) try: if not df.empty: df["client_time"] = pd.to_datetime(df["client_time"]) except Exception: tlog( f"[Error] aload_trades_by_portfolio_id({portfolio_id}) can't convert 'client_time' column to datetime" ) return df
async def run(self) -> bool: tickers = [] with ThreadPoolExecutor(max_workers=self.num_workers) as executor: with requests.Session() as session: count = self._get_count(session) loop = asyncio.get_event_loop() tasks = [ loop.run_in_executor( executor, self._fetch, *(session, page) ) for page in range(1, count // 50 + 1) ] for response in await asyncio.gather(*tasks): tickers += response tlog(f"loaded {len(tickers)} tickers") with ThreadPoolExecutor(max_workers=self.num_workers) as executor: with requests.Session() as session: loop = asyncio.get_event_loop() tasks = [ loop.run_in_executor( executor, self._fetch_symbol_details, # type: ignore *(session, ticker), ) for ticker in tickers ] info = [ response for response in await asyncio.gather(*tasks) if response is not None ] tlog(f"loaded {len(info)} ticker details") await asyncio.gather(*[self._update_ticker_details(i) for i in info]) return True
async def save(cls, df: DataFrame): pool = config.db_conn_pool async with pool.acquire() as con: for _, row in df.iterrows(): try: async with con.transaction(): _ = await con.fetchval( """ INSERT INTO trade_analysis (symbol, algo_run_id, gain_percentage, gain_value, r_units, start_tstamp, end_tstamp) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING trade_analysis_id """, row.symbol, row.algo_run_id, row.gain_percentage, row.gain_value, row.r_units, row.start_time.to_pydatetime(), row.end_time.to_pydatetime(), ) except Exception as e: tlog(f"[ERROR] inserting {row} resulted in exception {e}")
async def trades_handler(cls, msg): try: event = { "symbol": msg.symbol, "price": msg.price, "timestamp": pd.to_datetime(msg.timestamp), "volume": msg.size, "exchange": msg.exchange, "conditions": msg.conditions, "tape": msg.tape, "EV": "T", } cls.get_instance().queues[msg.symbol].put(event, timeout=1) except queue.Full as f: tlog( f"[EXCEPTION] process_message(): queue for {event['sym']} is FULL:{f}, sleeping for 2 seconds and re-trying." ) raise except Exception as e: tlog( f"[EXCEPTION] process_message(): exception of type {type(e).__name__} with args {e.args}" ) traceback.print_exc()
def get_historical_daily_from_polygon_by_range( api: tradeapi, symbols: List[str], start_date: date, end_date: date ) -> Dict[str, df]: """get ticker history""" _minute_history: Dict[str, df] = {} try: for symbol in symbols: retry = 5 _df = None while retry > 0: try: _df = api.polygon.historic_agg_v2( symbol, 1, "day", _from=str(start_date), to=str(end_date), ).df _df["vwap"] = 0.0 _df["average"] = 0.0 _minute_history[symbol] = ( pd.concat([_minute_history[symbol], _df]) if symbol in _minute_history else _df ) break except Exception as e: retry -= 1 continue except KeyboardInterrupt: tlog("KeyboardInterrupt") return _minute_history
async def run( self, symbol: str, shortable: bool, position: float, now: datetime, minute_history: pd.DataFrame, portfolio_value: float = None, debug: bool = False, backtesting: bool = False, ) -> Tuple[bool, Dict]: if position == 0.0 and await self.buy_signal(symbol, minute_history, now): size = await self.calc_amount(symbol, now) tlog( f"making purchase of {symbol} at {size} @ {minute_history.close[-1]}" ) return ( True, { "side": "buy", "qty": size, "type": "limit", "limit_price": minute_history.close[-1], }, ) elif position != 0.0: tlog("we need to sell!") # scale-out # sell if not picking up. pass return False, {}
async def liquidate(self): for symbol in trading_data.positions: if ( trading_data.positions[symbol] != 0 and trading_data.last_used_strategy[symbol].type == StrategyType.DAY_TRADE ): position = trading_data.positions[symbol] minute_index = self.minute_history[symbol][ "close" ].index.get_loc(self.now, method="nearest") price = self.minute_history[symbol]["close"][minute_index] tlog(f"[{self.end}]{symbol} liquidate {position} at {price}") db_trade = NewTrade( algo_run_id=trading_data.last_used_strategy[symbol].algo_run.run_id, # type: ignore symbol=symbol, qty=int(position) if int(position) > 0 else -int(position), operation="sell" if position > 0 else "buy", price=price, indicators={"liquidate": 1}, ) await db_trade.save( config.db_conn_pool, str(self.now.to_pydatetime()) )
async def scanner_runner(scanner: Scanner, queue: mp.Queue) -> None: try: while True: symbols = await scanner.run() for symbol in symbols: try: queue.put( json.dumps({ "symbol": symbol, "target_strategy_name": scanner.target_strategy_name, })) await asyncio.sleep(0) except Exception as e: tlog( f"[ERROR]Exception in scanner_runner({scanner.name}): exception of type {type(e).__name__} with args {e.args}" ) if scanner.recurrence: try: await asyncio.sleep(scanner.recurrence.total_seconds()) tlog(f"scanner {scanner.name} re-running") except asyncio.CancelledError: tlog( f"scanner_runner({scanner.name}) cancelled during sleep, closing scanner task" ) break else: break except asyncio.CancelledError: tlog( f"scanner_runner() cancelled, closing scanner task {scanner.name}") finally: tlog(f"scanner_runner {scanner.name} completed")
def get_batch_list(): @timeit async def get_batch_list_worker(): await create_db_connection() data = await AlgoRun.get_batches() print( tabulate( data, headers=["build", "batch_id", "strategy", "env", "start time"], )) try: if not asyncio.get_event_loop().is_closed(): asyncio.get_event_loop().close() loop = asyncio.new_event_loop() asyncio.set_event_loop(asyncio.new_event_loop()) loop.run_until_complete(get_batch_list_worker()) except KeyboardInterrupt: tlog("get_batch_list() - Caught KeyboardInterrupt") except Exception as e: tlog( f"get_batch_list() - exception of type {type(e).__name__} with args {e.args}" ) traceback.print_exc()