def test_filter(): scheduler = HistoricNetworkScheduler(0, 30 * 1000) network = scheduler.get_network() values = From(scheduler, [0.0, -3.2, 2.1, -2.9, 8.3, -5.7]) filt = Filter(network, values, lambda x: x >= 0.0) Do(network, filt, lambda: print(f'{filt.get_value()}')) scheduler.run() assert filt.get_value() == 8.3
async def _subscribe_trades_and_quotes(self): network = self.scheduler.get_network() for instrument in self.get_instruments(): symbol = instrument.get_exchange_instrument_code() self.instrument_trades[symbol] = MutableSignal() self.instrument_quotes[symbol] = MutableSignal() # magic: inject the bare Signal into the graph so we can # fire events on it without any downstream connections # yet made network.attach(self.instrument_trades[symbol]) network.attach(self.instrument_quotes[symbol]) subscribe_msg = { 'id': 1, 'method': 'trade.subscribe', 'params': [symbol] } messages = MutableSignal() json_messages = Map(network, messages, lambda x: json.loads(x)) incr_messages = Filter(network, json_messages, lambda x: x.get('type', None) == 'incremental') trade_lists = Map(network, incr_messages, lambda x: self.__extract_trades(x)) trades = FlatMap(self.scheduler, trade_lists) class TradeScheduler(Event): # noinspection PyShadowingNames def __init__(self, fh: PhemexFeedHandler, trades: Signal): self.fh = fh self.trades = trades def on_activate(self) -> bool: if self.trades.is_valid(): trade = self.trades.get_value() trade_symbol = trade.get_instrument().get_exchange_instrument_code() trade_signal = self.fh.instrument_trades[trade_symbol] self.fh.scheduler.schedule_update(trade_signal, trade) return True else: return False network.connect(trades, TradeScheduler(self, trades)) # noinspection PyShadowingNames async def do_subscribe(instrument, subscribe_msg, messages): async with websockets.connect(self.ws_uri) as sock: subscribe_msg_txt = json.dumps(subscribe_msg) self.logger.info(f'sending subscription request for {instrument.get_exchange_instrument_code()}') await sock.send(subscribe_msg_txt) while True: self.scheduler.schedule_update(messages, await sock.recv()) asyncio.ensure_future(do_subscribe(instrument, subscribe_msg, messages)) # we are now live self.scheduler.schedule_update(self.state, FeedHandlerState.LIVE)
def init(self, ctx: StrategyContext): self.ctx = ctx big_print_qty = float(ctx.getenv('BIG_PRINT_QTY')) self.trade_qty = float(ctx.getenv('CONTRACT_TRADE_QTY')) api_key = ctx.getenv('PHEMEX_API_KEY') api_secret = ctx.getenv('PHEMEX_API_SECRET') if not api_key: raise ValueError('missing PHEMEX_API_KEY') if not api_secret: raise ValueError('missing PHEMEX_API_SECRET') credentials = AuthCredentials(api_key, api_secret) exchange_instance = ctx.getenv('PHEMEX_INSTANCE', 'prod') if exchange_instance == 'prod': self.trading_conn = PhemexConnection(credentials) elif exchange_instance == 'test': self.trading_conn = PhemexConnection( credentials, api_url='https://testnet-api.phemex.com') else: raise ValueError( f'Unknown PHEMEX_INSTANCE value: {exchange_instance}') self.logger.info(f'Connected to Phemex {exchange_instance} instance') self.spot_feed = ctx.fh_registry.get_feed( f'coinbasepro:{exchange_instance}:BTC-USD') self.futures_feed = ctx.fh_registry.get_feed( f'phemex:{exchange_instance}:BTCUSD') self.logger.info( f'Connected to spot & futures {exchange_instance} feeds') network = self.ctx.get_network() # scan the spot market for large trades spot_trades = self.spot_feed.get_trades() Do( network, spot_trades, lambda: self.logger.info( f'Spot market trade: {spot_trades.get_value()}')) self.big_prints = Filter(network, spot_trades, lambda x: x.get_qty() >= big_print_qty) # compute 5 minute bins for the futures market and extract the volume field self.futures_trades = self.futures_feed.get_trades() buffer_5min = BufferWithTime(network, self.futures_trades, timedelta(minutes=5)) self.ohlc_5min = ComputeOHLC(network, buffer_5min) Do( network, self.ohlc_5min, lambda: self.logger.info( f'OHLC[5min]: {self.ohlc_5min.get_value()}')) self.volume = Map(network, self.ohlc_5min, lambda x: x.volume) # track the exponentially weighted moving average of the futures volume self.ewma = ExponentialMovingAverage(network, self.volume)
async def _subscribe_trades_and_quotes(self): network = self.scheduler.get_network() symbols = [] for instrument in self.get_instruments(): symbol = instrument.get_exchange_instrument_code() symbols.append(f'{symbol}') self.instrument_trades[symbol] = MutableSignal() self.instrument_quotes[symbol] = MutableSignal() self.instrument_order_books[symbol] = MutableSignal() # magic: inject the bare Signal into the graph so we can # fire events on it without any downstream connections # yet made network.attach(self.instrument_trades[symbol]) network.attach(self.instrument_quotes[symbol]) network.attach(self.instrument_order_books[symbol]) subscribe_msg = { 'type': 'subscribe', 'product_ids': symbols, 'channels': ['matches', 'heartbeat'] } messages = MutableSignal() json_messages = Map(network, messages, lambda x: json.loads(x)) match_messages = Filter(network, json_messages, lambda x: x.get('type', None) == 'match') trades = Map(network, match_messages, lambda x: self.__extract_trade(x)) class TradeScheduler(Event): def __init__(self, fh: CoinbaseProFeedHandler): self.fh = fh def on_activate(self) -> bool: if trades.is_valid(): trade = trades.get_value() trade_symbol = trade.get_instrument( ).get_exchange_instrument_code() trade_signal = self.fh.instrument_trades[trade_symbol] self.fh.scheduler.schedule_update(trade_signal, trade) return True else: return False network.connect(trades, TradeScheduler(self)) async with websockets.connect(self.ws_uri) as sock: self.logger.info('Sending subscription request for all products') await sock.send(json.dumps(subscribe_msg)) self.scheduler.schedule_update(self.state, FeedHandlerState.LIVE) while True: self.scheduler.schedule_update(messages, await sock.recv())
def start(self): network = self.scheduler.get_network() messages = MutableSignal() json_messages = Map(network, messages, lambda x: json.loads(x)) json_messages = Filter( network, json_messages, lambda x: x.get('type', None) in ['incremental', 'snapshot']) class OrderEventScheduler(Event): # noinspection PyShadowingNames def __init__(self, sub: AccountOrderPositionSubscriber, json_messages: Signal): self.sub = sub self.json_messages = json_messages def on_activate(self) -> bool: if self.json_messages.is_valid(): msg = self.json_messages.get_value() accounts = msg['accounts'] for account in accounts: orders = account['orders'] for order in orders: order_event = OrderEvent() self.sub.scheduler.schedule_update( self.sub.order_events, order_event) return True else: return False network.connect(json_messages, OrderEventScheduler(self, json_messages)) # noinspection PyShadowingNames async def do_subscribe(): async with websockets.connect(self.ws_uri) as sock: self.logger.info( f'sending Account-Order-Position subscription request') auth_msg = self.auth.get_user_auth_message() await sock.send(auth_msg) error_msg = await sock.recv() error_struct = json.loads(error_msg) if error_struct['error'] is not None: raise ConnectionError( f'Unable to authenticate: {error_msg}') aop_sub_msg = { 'id': 2, 'method': 'aop.subscribe', 'params': [] } await sock.send(json.dumps(aop_sub_msg)) while True: self.scheduler.schedule_update(messages, await sock.recv()) asyncio.ensure_future(do_subscribe())
def init(self, ctx: StrategyContext): self.ctx = ctx big_print_qty = float(ctx.getenv('BIG_PRINT_QTY')) self.trade_qty = float(ctx.getenv('CONTRACT_TRADE_QTY')) exchange_instance = ctx.getenv('EXCHANGE_INSTANCE', 'prod') op_uri = f'phemex:{exchange_instance}' self.order_placer = ctx.get_order_placer_service().get_order_placer( op_uri) self.logger.info(f'Connected to Phemex {exchange_instance} instance') network = self.ctx.get_network() # scan the spot market for large trades btc_usd_spot = self.ctx.get_instrument_cache( ).get_crypto_exchange_instrument('CoinbasePro', 'BTC-USD') spot_trades = self.ctx.get_marketdata_service().get_trades( btc_usd_spot) Do( network, spot_trades, lambda: self.logger.info( f'Spot market trade: {spot_trades.get_value()}')) self.big_prints = Filter(network, spot_trades, lambda x: x.get_qty() >= big_print_qty) # compute 5 minute bins for the futures market and extract the volume field btc_usd_future = self.ctx.get_instrument_cache( ).get_crypto_exchange_instrument('Phemex', 'BTCUSD') self.futures_trades = self.ctx.get_marketdata_service().get_trades( btc_usd_future) buffer_5min = BufferWithTime(self.ctx.get_scheduler(), self.futures_trades, timedelta(minutes=5)) self.ohlc_5min = ComputeOHLC(network, buffer_5min) Do( network, self.ohlc_5min, lambda: self.logger.info( f'OHLC[5min]: {self.ohlc_5min.get_value()}')) self.volume = Map(network, self.ohlc_5min, lambda x: x.volume) # subscribe to top of book self.futures_book = self.ctx.get_marketdata_service().get_order_books( btc_usd_future) Do( network, self.futures_book, lambda: self.logger.info( f'Futures bid/ask: ' f'{self.futures_book.get_value().get_best_bid()} / ' f'{self.futures_book.get_value().get_best_ask()}')) # track the exponentially weighted moving average of the futures volume self.ewma = ExponentialMovingAverage(network, self.volume)
def get_marks(self, instrument: ExchangeInstrument) -> Signal: economics = instrument.get_instrument().get_economics() if isinstance(economics, FutureContract): economics = economics.get_underlier().get_economics() if isinstance(economics, CurrencyPair): ccy_code = economics.get_base_currency().get_currency_code() symbol = f'.M{ccy_code}' cash_instrument = self.instrument_cache.get_or_create_cash_instrument( ccy_code) else: raise ValueError( 'Unable to get marks: expected CurrencyPair or FutureContract on CurrencyPair' ) network = self.scheduler.get_network() marks = self.instrument_marks.get(symbol) if marks is None: marks = MutableSignal() network.attach(marks) self.instrument_marks[symbol] = marks subscribe_msg = { 'id': 1, 'method': 'tick.subscribe', 'params': [symbol] } messages = MutableSignal() json_messages = Map(network, messages, lambda x: json.loads(x)) tick_messages = Filter(network, json_messages, lambda x: 'tick' in x) ticks = Map( network, tick_messages, lambda x: self.__extract_tick( x, cash_instrument.get_instrument())) Pipe(self.scheduler, ticks, marks) asyncio.ensure_future( websocket_subscribe_with_retry(self.ws_uri, self.timeout, self.logger, subscribe_msg, self.scheduler, messages, symbol, 'ticks')) return marks
async def _subscribe_trades_and_quotes(self): network = self.scheduler.get_network() symbols = [] for instrument in self.get_instruments(): symbol = instrument.get_exchange_instrument_code() symbols.append(f'{symbol.lower()}@aggTrade') self.instrument_trades[symbol] = MutableSignal() self.instrument_quotes[symbol] = MutableSignal() # magic: inject the bare Signal into the graph so we can # fire events on it without any downstream connections # yet made network.attach(self.instrument_trades[symbol]) network.attach(self.instrument_quotes[symbol]) messages = MutableSignal() json_messages = Map(network, messages, lambda x: json.loads(x)) trade_messages = Filter(network, json_messages, lambda x: 'data' in x) trades = Map(network, trade_messages, lambda x: self.__extract_trade(x)) class TradeScheduler(Event): def __init__(self, fh: BinanceFeedHandler): self.fh = fh def on_activate(self) -> bool: if trades.is_valid(): trade = trades.get_value() trade_symbol = trade.get_instrument( ).get_exchange_instrument_code() trade_signal = self.fh.instrument_trades[trade_symbol] self.fh.scheduler.schedule_update(trade_signal, trade) return True else: return False network.connect(trades, TradeScheduler(self)) async with websockets.connect(self.ws_uri) as sock: ndx = 1 n = 250 symbols_chunked = [ symbols[i:i + n] for i in range(0, len(symbols), n) ] for symbols in symbols_chunked: self.logger.info( f'Sending subscription request for {len(symbols)} symbols: {symbols}' ) subscribe_msg = { "method": "SUBSCRIBE", "params": symbols, "id": ndx } await sock.send(json.dumps(subscribe_msg)) ndx = ndx + 1 self.scheduler.schedule_update(self.state, FeedHandlerState.LIVE) while True: self.scheduler.schedule_update(messages, await sock.recv())
def init(self, ctx: StrategyContext): self.ctx = ctx big_print_qty = float(ctx.getenv('BIG_PRINT_QTY')) self.trade_qty = float(ctx.getenv('CONTRACT_TRADE_QTY')) api_key = ctx.getenv('PHEMEX_API_KEY') api_secret = ctx.getenv('PHEMEX_API_SECRET') if not api_key: raise ValueError('missing PHEMEX_API_KEY') if not api_secret: raise ValueError('missing PHEMEX_API_SECRET') credentials = AuthCredentials(api_key, api_secret) exchange_instance = ctx.getenv('PHEMEX_INSTANCE', 'prod') if exchange_instance == 'prod': self.trading_conn = PhemexConnection(credentials) elif exchange_instance == 'test': self.trading_conn = PhemexConnection( credentials, api_url='https://testnet-api.phemex.com') else: raise ValueError( f'Unknown PHEMEX_INSTANCE value: {exchange_instance}') self.logger.info(f'Connected to Phemex {exchange_instance} instance') network = self.ctx.get_network() # subscribe to AOP messages auth = WebsocketAuthenticator(api_key, api_secret) aop_sub = AccountOrderPositionSubscriber(auth, ctx.get_scheduler(), exchange_instance) aop_sub.start() # scan the spot market for large trades btc_usd_spot = self.ctx.get_instrument_cache().get_exchange_instrument( 'CoinbasePro', 'BTC-USD') spot_trades = self.ctx.get_marketdata_service().get_trades( btc_usd_spot) Do( network, spot_trades, lambda: self.logger.info( f'Spot market trade: {spot_trades.get_value()}')) self.big_prints = Filter(network, spot_trades, lambda x: x.get_qty() >= big_print_qty) # compute 5 minute bins for the futures market and extract the volume field btc_usd_future = self.ctx.get_instrument_cache( ).get_exchange_instrument('Phemex', 'BTCUSD') self.futures_trades = self.ctx.get_marketdata_service().get_trades( btc_usd_future) buffer_5min = BufferWithTime(network, self.futures_trades, timedelta(minutes=5)) self.ohlc_5min = ComputeOHLC(network, buffer_5min) Do( network, self.ohlc_5min, lambda: self.logger.info( f'OHLC[5min]: {self.ohlc_5min.get_value()}')) self.volume = Map(network, self.ohlc_5min, lambda x: x.volume) # subscribe to top of book self.futures_book = self.ctx.get_marketdata_service().get_order_books( btc_usd_future) Do( network, self.futures_book, lambda: self.logger.info( f'Futures bid/ask: ' f'{self.futures_book.get_value().get_best_bid()} / ' f'{self.futures_book.get_value().get_best_ask()}')) # track the exponentially weighted moving average of the futures volume self.ewma = ExponentialMovingAverage(network, self.volume)
async def _subscribe_trades_and_quotes(self): network = self.scheduler.get_network() symbols = [] for instrument in self.get_instruments(): symbol = instrument.get_exchange_instrument_code() if symbol == self.include_symbol or self.include_symbol == '*': symbols.append(f'{symbol}') self.instrument_trades[symbol] = MutableSignal() self.instrument_order_book_events[symbol] = MutableSignal() self.instrument_order_books[symbol] = OrderBookBuilder( network, self.instrument_order_book_events[symbol]) # magic: inject the bare Signal into the graph so we can # fire events on it without any downstream connections # yet made network.attach(self.instrument_trades[symbol]) network.attach(self.instrument_order_book_events[symbol]) network.attach(self.instrument_order_books[symbol]) subscribe_msg = { 'type': 'subscribe', 'product_ids': symbols, 'channels': ['matches', 'level2', 'heartbeat'] } messages = MutableSignal() json_messages = Map(network, messages, lambda x: json.loads(x)) match_messages = Filter(network, json_messages, lambda x: x.get('type', None) == 'match') book_messages = Filter( network, json_messages, lambda x: x.get('type', None) in {'snapshot', 'l2update'}) trades = Map(network, match_messages, lambda x: self.__extract_trade(x)) books = Map(network, book_messages, lambda x: self.__extract_order_book_event(x)) class TradeScheduler(Event): def __init__(self, fh: CoinbaseProFeedHandler): self.fh = fh def on_activate(self) -> bool: if trades.is_valid(): trade = trades.get_value() trade_symbol = trade.get_instrument( ).get_exchange_instrument_code() trade_signal = self.fh.instrument_trades[trade_symbol] self.fh.scheduler.schedule_update(trade_signal, trade) return True else: return False network.connect(trades, TradeScheduler(self)) class OrderBookScheduler(Event): def __init__(self, fh: CoinbaseProFeedHandler): self.fh = fh def on_activate(self) -> bool: if books.is_valid(): obe = books.get_value() obe_symbol = obe.get_instrument( ).get_exchange_instrument_code() obe_signal = self.fh.instrument_order_book_events[ obe_symbol] self.fh.scheduler.schedule_update(obe_signal, obe) return True else: return False network.connect(books, OrderBookScheduler(self)) asyncio.ensure_future( websocket_subscribe_with_retry(self.ws_uri, self.timeout, self.logger, subscribe_msg, self.scheduler, messages, 'all products', 'global')) # we are now live self.scheduler.schedule_update(self.state, FeedHandlerState.LIVE)
def start(self): network = self.scheduler.get_network() messages = MutableSignal() json_messages = Map(network, messages, lambda x: json.loads(x)) json_messages = Filter(network, json_messages, lambda x: x.get('type', None) == 'incremental') class OrderEventScheduler(Event): # noinspection PyShadowingNames def __init__(self, sub: OrderEventSubscriber, json_messages: Signal): self.sub = sub self.json_messages = json_messages def on_activate(self) -> bool: if self.json_messages.is_valid(): msg = self.json_messages.get_value() orders = msg['orders'] for order_msg in orders: order_id = order_msg['orderID'] cl_ord_id = order_msg['clOrdID'] exec_id = order_msg['execID'] last_px = order_msg['execPriceEp'] / 10000 last_qty = order_msg['execQty'] order = self.sub.oms.get_order_by_cl_ord_id(cl_ord_id) if order is None: if cl_ord_id is None or cl_ord_id == '': self.sub.logger.warning( f'Received message from exchange with missing clOrdID, ' f'orderID={order_id}') else: self.sub.logger.warning( f'Received message from exchange for unknown ' f'clOrdID={cl_ord_id}, orderID={order_id}') return False elif order.get_order_id() is None: self.sub.logger.info( f'OMS order missing orderID; patching from clOrdID={cl_ord_id}' ) order.set_order_id(order_id) if order_msg['ordStatus'] == 'New': self.sub.oms.new(order, exec_id) elif order_msg['ordStatus'] == 'Canceled': self.sub.oms.apply_cancel(order, exec_id) elif order_msg[ 'ordStatus'] == 'PartiallyFilled' or order_msg[ 'ordStatus'] == 'Filled': self.sub.oms.apply_fill(order, last_qty, last_px, exec_id) return True else: return False network.connect(json_messages, OrderEventScheduler(self, json_messages)) # noinspection PyShadowingNames,PyBroadException async def do_subscribe(): while True: try: async with websockets.connect(self.ws_uri) as sock: self.logger.info( 'sending Account-Order-Position subscription request for orders' ) auth_msg = self.auth.get_user_auth_message(1) await sock.send(auth_msg) error_msg = await sock.recv() error_struct = json.loads(error_msg) if error_struct['error'] is not None: raise ConnectionError( f'Unable to authenticate: {error_msg}') aop_sub_msg = { 'id': 2, 'method': 'aop.subscribe', 'params': [] } await sock.send(json.dumps(aop_sub_msg)) while True: try: self.scheduler.schedule_update( messages, await sock.recv()) except BaseException as error: self.logger.error( f'disconnected; attempting to reconnect after {self.timeout} ' f'seconds: {error}') await asyncio.sleep(self.timeout) # exit inner loop break except socket.gaierror as error: self.logger.error( f'failed with socket error; attempting to reconnect after {self.timeout} ' f'seconds: {error}') await asyncio.sleep(self.timeout) continue except ConnectionRefusedError as error: self.logger.error( f'connection refused; attempting to reconnect after {self.timeout} ' f'seconds: {error}') await asyncio.sleep(self.timeout) continue except BaseException as error: self.logger.error( f'unknown connection error; attempting to reconnect after {self.timeout} ' f'seconds: {error}') await asyncio.sleep(self.timeout) continue asyncio.ensure_future(do_subscribe())
async def _subscribe_trades_and_quotes(self): network = self.scheduler.get_network() for instrument in self.get_instruments(): symbol = instrument.get_exchange_instrument_code() if symbol == self.include_symbol or self.include_symbol == '*': self.instrument_trades[symbol] = MutableSignal() self.instrument_order_book_events[symbol] = MutableSignal() self.instrument_order_books[symbol] = OrderBookBuilder( network, self.instrument_order_book_events[symbol]) # magic: inject the bare Signal into the graph so we can # fire events on it without any downstream connections network.attach(self.instrument_trades[symbol]) network.attach(self.instrument_order_book_events[symbol]) network.attach(self.instrument_order_books[symbol]) trade_subscribe_msg = { 'id': 1, 'method': 'trade.subscribe', 'params': [symbol] } trade_messages = MutableSignal() trade_json_messages = Map(network, trade_messages, lambda x: json.loads(x)) trade_incr_messages = Filter( network, trade_json_messages, lambda x: x.get('type', None) == 'incremental') trade_lists = Map(network, trade_incr_messages, lambda x: self.__extract_trades(x)) trades = FlatMap(self.scheduler, trade_lists) class TradeScheduler(Event): # noinspection PyShadowingNames def __init__(self, fh: PhemexFeedHandler, trades: Signal): self.fh = fh self.trades = trades def on_activate(self) -> bool: if self.trades.is_valid(): trade = self.trades.get_value() trade_symbol = trade.get_instrument( ).get_exchange_instrument_code() trade_signal = self.fh.instrument_trades[ trade_symbol] self.fh.scheduler.schedule_update( trade_signal, trade) return True else: return False network.connect(trades, TradeScheduler(self, trades)) orderbook_subscribe_msg = { 'id': 2, 'method': 'orderbook.subscribe', 'params': [symbol] } obe_messages = MutableSignal() obe_json_messages = Map(network, obe_messages, lambda x: json.loads(x)) obe_json_messages = Filter( network, obe_json_messages, lambda x: x.get('type', None) in ['incremental', 'snapshot']) order_book_events = Map( network, obe_json_messages, lambda x: self.__extract_order_book_event(x)) class OrderBookEventScheduler(Event): # noinspection PyShadowingNames def __init__(self, fh: PhemexFeedHandler, order_book_events: Signal): self.fh = fh self.order_book_events = order_book_events def on_activate(self) -> bool: if self.order_book_events.is_valid(): obe = self.order_book_events.get_value() obe_symbol = obe.get_instrument( ).get_exchange_instrument_code() obe_signal = self.fh.instrument_order_book_events[ obe_symbol] self.fh.scheduler.schedule_update(obe_signal, obe) return True else: return False network.connect( order_book_events, OrderBookEventScheduler(self, order_book_events)) asyncio.ensure_future( websocket_subscribe_with_retry(self.ws_uri, self.timeout, self.logger, trade_subscribe_msg, self.scheduler, trade_messages, symbol, 'trade')) asyncio.ensure_future( websocket_subscribe_with_retry(self.ws_uri, self.timeout, self.logger, orderbook_subscribe_msg, self.scheduler, obe_messages, symbol, 'order book')) # we are now live self.scheduler.schedule_update(self.state, FeedHandlerState.LIVE)
async def main(): scheduler = RealtimeNetworkScheduler() network = scheduler.get_network() values = From(scheduler, [0.0, -3.2, 2.1, -2.9, 8.3, -5.7]) filt = Filter(network, values, lambda x: x >= 0.0) Do(network, filt, lambda: print(f"{filt.get_value()}"))