def _symbol_endpoint_prepare(cls, ep: RestEndpoint) -> Union[List[str], str]: if ep.routes.currencies: ret = cls.http_sync.read(ep.route('currencies'), json=True, uuid=cls.id) return [ ep.route('instruments').format(currency) for currency in ret['data']['currencies'] ] return ep.route('instruments')
def _symbol_endpoint_prepare(cls, ep: RestEndpoint) -> Union[List[str], str]: """ override if a specific exchange needs to do something first, like query an API to get a list of currencies, that are then used to build the list of symbol endpoints """ return ep.route('instruments')
class BinanceUS(Binance, BinanceUSRestMixin): id = BINANCE_US websocket_endpoints = [WebsocketEndpoint('wss://stream.binance.us:9443')] rest_endpoints = [ RestEndpoint('https://api.binance.us', routes=Routes('/api/v3/exchangeInfo', l2book='/api/v3/depth?symbol={}&limit={}')) ]
class FTXTR(FTX, FTXTRRestMixin): id = FTX_TR websocket_endpoints = [WebsocketEndpoint("wss://ftxtr.com/ws/", options={"compression": None})] rest_endpoints = [RestEndpoint("https://ftxtr.com", routes=Routes("/api/markets"))] websocket_channels = { L2_BOOK: "orderbook", TRADES: "trades", TICKER: "ticker", ORDER_INFO: "orders", FILLS: "fills", }
class FTXUS(FTX, FTXUSRestMixin): id = FTX_US websocket_endpoints = [ WebsocketEndpoint('wss://ftx.us/ws/', options={'compression': None}) ] rest_endpoints = [ RestEndpoint('https://ftx.us', routes=Routes('/api/markets')) ] websocket_channels = { L2_BOOK: 'orderbook', TRADES: 'trades', TICKER: 'ticker', ORDER_INFO: 'orders', FILLS: 'fills', }
class HitBTC(Bequant): id = HITBTC websocket_channels = { BALANCES: 'subscribeBalance', TRANSACTIONS: 'subscribeTransactions', ORDER_INFO: 'subscribeReports', L2_BOOK: 'subscribeOrderbook', TRADES: 'subscribeTrades', TICKER: 'subscribeTicker', CANDLES: 'subscribeCandles' } websocket_endpoints = [ WebsocketEndpoint('wss://api.hitbtc.com/api/2/ws/public', channel_filter=(websocket_channels[L2_BOOK], websocket_channels[TRADES], websocket_channels[TICKER], websocket_channels[CANDLES])), WebsocketEndpoint('wss://api.hitbtc.com/api/2/ws/trading', channel_filter=(websocket_channels[ORDER_INFO],)), WebsocketEndpoint('wss://api.hitbtc.com/api/2/ws/account', channel_filter=(websocket_channels[BALANCES], websocket_channels[TRANSACTIONS])), ] rest_endpoints = [RestEndpoint('https://api.hitbtc.com', routes=Routes('/api/2/public/symbol'))]
class AscendEXFutures(AscendEX): """ Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#introducing-futures-pro-v2-apis """ id = ASCENDEX_FUTURES websocket_channels = { **AscendEX.websocket_channels, } # Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#how-to-connect # noinspection PyTypeChecker websocket_endpoints = [ WebsocketEndpoint( 'wss://ascendex.com:443/api/pro/v2/stream', channel_filter=( websocket_channels[L2_BOOK], websocket_channels[TRADES], ), sandbox='wss://api-test.ascendex-sandbox.com:443/api/pro/v2/stream' ) ] # Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#futures-contracts-info rest_endpoints = [ RestEndpoint('https://ascendex.com', routes=Routes('/api/pro/v2/futures/contract'), sandbox='https://api-test.ascendex-sandbox.com') ] @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: # Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#futures-contracts-info ret = {} info = defaultdict(dict) for entry in data['data']: # Only "Normal" status symbols are tradeable if entry['status'] == 'Normal': s = Symbol(re.sub(entry['settlementAsset'], '', entry['displayName']), entry['settlementAsset'], type=PERPETUAL) ret[s.normalized] = entry['symbol'] info['tick_size'][ s.normalized] = entry['priceFilter']['tickSize'] info['instrument_type'][s.normalized] = s.type return ret, info
class Poloniex(Feed, PoloniexRestMixin): id = POLONIEX websocket_endpoints = [WebsocketEndpoint('wss://api2.poloniex.com')] rest_endpoints = [ RestEndpoint('https://poloniex.com', routes=Routes('/public?command=returnTicker')) ] _channel_map = {} websocket_channels = { L2_BOOK: L2_BOOK, TRADES: TRADES, TICKER: 1002, } request_limit = 6 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = {'instrument_type': {}} for symbol in data: cls._channel_map[data[symbol]['id']] = symbol std = symbol.replace("STR", "XLM") quote, base = std.split("_") s = Symbol(base, quote) ret[s.normalized] = symbol info['instrument_type'][s.normalized] = s.type return ret, info def __reset(self): self._l2_book = {} self.seq_no = {} async def _ticker(self, msg: dict, timestamp: float): """ Format: currencyPair, last, lowestAsk, highestBid, percentChange, baseVolume, quoteVolume, isFrozen, 24hrHigh, 24hrLow, postOnly, maintenance mode The postOnly field indicates that new orders posted to the market must be non-matching orders (no taker orders). Any orders that would match will be rejected. Maintenance mode indicates that maintenace is being performed and orders will be rejected """ pair_id, _, ask, bid, _, _, _, _, _, _, _, _ = msg if pair_id not in self._channel_map: # Ignore new trading pairs that are added during long running sessions return pair = self.exchange_symbol_to_std_symbol(self._channel_map[pair_id]) t = Ticker(self.id, pair, Decimal(bid), Decimal(ask), None, raw=msg) await self.callback(TICKER, t, timestamp) async def _book(self, msg: dict, chan_id: int, timestamp: float): delta = {BID: [], ASK: []} msg_type = msg[0][0] pair = None # initial update (i.e. snapshot) if msg_type == 'i': delta = None pair = msg[0][1]['currencyPair'] pair = self.exchange_symbol_to_std_symbol(pair) self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) # 0 is asks, 1 is bids order_book = msg[0][1]['orderBook'] for index, side in enumerate([ASK, BID]): for key in order_book[index]: amount = Decimal(order_book[index][key]) price = Decimal(key) self._l2_book[pair].book[side][price] = amount else: pair = self._channel_map[chan_id] pair = self.exchange_symbol_to_std_symbol(pair) for update in msg: msg_type = update[0] # order book update if msg_type == 'o': side = ASK if update[1] == 0 else BID price = Decimal(update[2]) amount = Decimal(update[3]) if amount == 0: delta[side].append((price, 0)) del self._l2_book[pair].book[side][price] else: delta[side].append((price, amount)) self._l2_book[pair].book[side][price] = amount elif msg_type == 't': # index 1 is trade id, 2 is side, 3 is price, 4 is amount, 5 is timestamp, 6 is timestamp ms _, order_id, _, price, amount, server_ts, _ = update price = Decimal(price) amount = Decimal(amount) t = Trade(self.id, pair, BUY if update[2] == 1 else SELL, amount, price, float(server_ts), id=order_id, raw=msg) await self.callback(TRADES, t, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg) await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, delta=delta, raw=msg) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'error' in msg: LOG.error("%s: Error from exchange: %s", self.id, msg) return chan_id = msg[0] if chan_id == 1002: # the ticker channel doesn't have sequence ids # so it should be None, except for the subscription # ack, in which case its 1 seq_id = msg[1] if seq_id is None and self._channel_map[ msg[2][0]] in self.subscription[1002]: await self._ticker(msg[2], timestamp) elif chan_id < 1000: # order book updates - the channel id refers to # the trading pair being updated seq_no = msg[1] if chan_id not in self.seq_no: self.seq_no[chan_id] = seq_no elif self.seq_no[chan_id] + 1 != seq_no and msg[2][0][0] != 'i': LOG.warning( "%s: missing sequence number. Received %d, expected %d", self.id, seq_no, self.seq_no[chan_id] + 1) raise MissingSequenceNumber self.seq_no[chan_id] = seq_no if msg[2][0][0] == 'i': del self.seq_no[chan_id] symbol = self._channel_map[chan_id] if symbol in self._trade_book_symbols: await self._book(msg[2], chan_id, timestamp) elif chan_id == 1010: # heartbeat - ignore pass else: LOG.warning('%s: Invalid message type %s', self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() self._trade_book_symbols = [] for chan in self.subscription: if chan == L2_BOOK or chan == TRADES: for symbol in self.subscription[chan]: if symbol in self._trade_book_symbols: continue self._trade_book_symbols.append(symbol) await conn.write( json.dumps({ "command": "subscribe", "channel": symbol })) else: await conn.write( json.dumps({ "command": "subscribe", "channel": chan })) self._trade_book_symbols = set(self._trade_book_symbols)
class Bithumb(Feed): ''' Before you use this bithumb implementation, you should know that this is exchange's API is pretty terrible. For some unknown reason, bithumb's api_info page lists all their KRW symbols as USDT. Probably because they bought the exchange and copied everything but didn't bother to update the reference data. We'll just assume that anything USDT is actually KRW. A search on their exchange page shows that there is no USDT symbols available. Please be careful when referencing their api_info page ''' id = BITHUMB websocket_endpoints = [ WebsocketEndpoint('wss://pubwss.bithumb.com/pub/ws') ] rest_endpoints = [ RestEndpoint('https://api.bithumb.com', routes=Routes( ['/public/ticker/ALL_BTC', '/public/ticker/ALL_KRW'])) ] websocket_channels = { # L2_BOOK: 'orderbookdepth', <-- technically the exchange supports orderbooks but it only provides orderbook deltas, there is # no way to synchronize against a rest snapshot, nor request/obtain an orderbook via the websocket, so this isn't really useful TRADES: 'transaction', } @classmethod def timestamp_normalize(cls, ts: dt) -> float: return (ts - timedelta(hours=9)).timestamp() # Override symbol_mapping class method, because this bithumb is a very special case. # There is no actual page in the API for reference info. # Need to query the ticker endpoint by quote currency for that info # To qeury the ticker endpoint, you need to know which quote currency you want. So far, seems like the exhcnage # only offers KRW and BTC as quote currencies. @classmethod def symbol_mapping(cls, refresh=False) -> Dict: if Symbols.populated(cls.id) and not refresh: return Symbols.get(cls.id)[0] try: data = {} for ep in cls.rest_endpoints[0].route('instruments'): ret = cls.http_sync.read(ep, json=True, uuid=cls.id) if 'BTC' in ep: data['BTC'] = ret else: data['KRW'] = ret syms, info = cls._parse_symbol_data(data) Symbols.set(cls.id, syms, info) return syms except Exception as e: LOG.error("%s: Failed to parse symbol information: %s", cls.id, str(e), exc_info=True) raise @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = {'instrument_type': {}} for quote_curr, response in data.items(): bases = response['data'] for base_curr in bases.keys(): if base_curr == 'date': continue s = Symbol(base_curr, quote_curr) ret[s.normalized] = f"{base_curr}_{quote_curr}" info['instrument_type'][s.normalized] = s.type return ret, info def __init__(self, max_depth=30, **kwargs): super().__init__(max_depth=max_depth, **kwargs) async def _trades(self, msg: dict, rtimestamp: float): ''' { "type": "transaction", "content": { "list": [ { "symbol": "BTC_KRW", // currency code "buySellGb": "1", // type of contract (1: sale contract, 2: buy contract) "contPrice": "10579000", // execution price "contQty": "0.01", // number of contracts "contAmt": "105790.00", // execution amount "contDtm": "2020-01-29 12:24:18.830039", // Signing time "updn": "dn" // comparison with the previous price: up-up, dn-down } ] } } ''' trades = msg.get('content', {}).get('list', []) for trade in trades: # API ref list uses '-', but market data returns '_' symbol = self.exchange_symbol_to_std_symbol(trade['symbol']) timestamp = self.timestamp_normalize(trade['contDtm']) price = Decimal(trade['contPrice']) quantity = Decimal(trade['contQty']) side = BUY if trade['buySellGb'] == '2' else SELL t = Trade(self.id, symbol, side, quantity, price, timestamp, raw=trade) await self.callback(TRADES, t, rtimestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) msg_type = msg.get('type', None) if msg_type == 'transaction': await self._trades(msg, timestamp) elif msg_type is None and msg.get('status', None) == '0000': return else: LOG.warning("%s: Unexpected message received: %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): if self.subscription: for chan in self.subscription: await conn.write( json.dumps({ "type": chan, "symbols": [symbol for symbol in self.subscription[chan]] # API ref list uses '-', but subscription requires '_' }))
class IndependentReserve(Feed): id = INDEPENDENT_RESERVE websocket_endpoints = [WebsocketEndpoint('wss://websockets.independentreserve.com')] rest_endpoints = [RestEndpoint('https://api.independentreserve.com', routes=Routes(['/Public/GetValidPrimaryCurrencyCodes', '/Public/GetValidSecondaryCurrencyCodes'], l3book='/Public/GetAllOrders?primaryCurrencyCode={}&secondaryCurrencyCode={}'))] websocket_channels = { L3_BOOK: 'orderbook-{}', TRADES: 'ticker-{}', } request_limit = 1 @classmethod def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) bases, quotes = data for base in bases: for quote in quotes: sym = Symbol(base.upper().replace('XBT', 'BTC'), quote.upper()) info['instrument_type'][sym.normalized] = sym.type ret[sym.normalized] = f"{base.lower()}-{quote.lower()}" return ret, info def __reset(self): self._l3_book = {} self._order_ids = defaultdict(dict) self._sequence_no = {} async def _trade(self, msg: dict, timestamp: float): ''' { 'Channel': 'ticker-eth-aud', 'Nonce': 78, 'Data': { 'TradeGuid': '6d1c2e90-592a-409c-a8d8-58b2d25e0b0b', 'Pair': 'eth-aud', 'TradeDate': datetime.datetime(2022, 1, 31, 8, 28, 26, 552573, tzinfo=datetime.timezone(datetime.timedelta(seconds=39600))), 'Price': Decimal('3650.81'), 'Volume': Decimal('0.543'), 'BidGuid': '0430e003-c35e-410e-85f5-f0bb5c40193b', 'OfferGuid': '559c1dd2-e681-4efc-b49b-14a07c069de4', 'Side': 'Sell' }, 'Time': 1643578106584, 'Event': 'Trade' } ''' t = Trade( self.id, self.exchange_symbol_to_std_symbol(msg['Data']['Pair']), SELL if msg['Data']['Side'] == 'Sell' else BUY, Decimal(msg['Data']['Volume']), Decimal(msg['Data']['Price']), self.timestamp_normalize(msg['Data']['TradeDate']), id=msg['Data']['TradeGuid'], raw=msg ) await self.callback(TRADES, t, timestamp) async def _book(self, msg: dict, timestamp: float): ''' { 'Channel': 'orderbook-xbt', 'Nonce': 65605, 'Data': { 'OrderType': 'LimitBid', 'OrderGuid': 'fee7094c-1921-44b7-8d8d-8b6e1cedb270' }, 'Time': 1643931382903, 'Event': 'OrderCanceled' } { 'Channel': 'orderbook-xbt', 'Nonce': 65606, 'Data': { 'OrderType': 'LimitOffer', 'OrderGuid': '22a72137-9829-4e6c-b265-a38714256877', 'Price': { 'aud': Decimal('51833.41'), 'usd': Decimal('37191.23'), 'nzd': Decimal('55836.92'), 'sgd': Decimal('49892.59') }, 'Volume': Decimal('0.09') }, 'Time': 1643931382903, 'Event': 'NewOrder' } ''' seq_no = msg['Nonce'] base = msg['Channel'].split('-')[-1] delta = {BID: [], ASK: []} for symbol in self.subscription[self.std_channel_to_exchange(L3_BOOK)]: if symbol.startswith(base): quote = symbol.split('-')[-1] instrument = self.exchange_symbol_to_std_symbol(f"{base}-{quote}") if instrument in self._sequence_no and self._sequence_no[instrument] + 1 != seq_no: raise MissingSequenceNumber self._sequence_no[instrument] = seq_no if instrument not in self._l3_book: await self._snapshot(base, quote) if msg['Event'] == 'OrderCanceled': uuid = msg['Data']['OrderGuid'] if uuid in self._order_ids[instrument]: price, side = self._order_ids[instrument][uuid] if price in self._l3_book[instrument].book[side] and uuid in self._l3_book[instrument].book[side][price]: del self._l3_book[instrument].book[side][price][uuid] if len(self._l3_book[instrument].book[side][price]) == 0: del self._l3_book[instrument].book[side][price] delta[side].append((uuid, price, 0)) del self._order_ids[instrument][uuid] else: # during snapshots we might get cancelation messages that have already been removed # from the snapshot, so we don't have anything to process, and we should not call the client callback continue elif msg['Event'] == 'NewOrder': uuid = msg['Data']['OrderGuid'] price = msg['Data']['Price'][quote] size = msg['Data']['Volume'] side = BID if msg['Data']['OrderType'].endswith('Bid') else ASK self._order_ids[instrument][uuid] = (price, side) if price in self._l3_book[instrument].book[side]: self._l3_book[instrument].book[side][price][uuid] = size else: self._l3_book[instrument].book[side][price] = {uuid: size} delta[side].append((uuid, price, size)) elif msg['event'] == 'OrderChanged': uuid = msg['Data']['OrderGuid'] size = msg['Data']['Volume'] side = BID if msg['Data']['OrderType'].endswith('Bid') else ASK if uuid in self._order_ids[instrument]: price, side = self._order_ids[instrument][uuid] if size == 0: del self._l3_book[instrument][side][price][uuid] if len(self._l3_book[instrument][side][price]) == 0: del self._l3_book[instrument][side][price] else: self._l3_book[instrument][side][price][uuid] = size del self._order_ids[instrument][uuid] delta[side].append((uuid, price, size)) else: continue else: raise ValueError("%s: Invalid OrderBook event message of type %s", self.id, msg) await self.book_callback(L3_BOOK, self._l3_book[instrument], timestamp, raw=msg, sequence_number=seq_no, delta=delta, timestamp=msg['Time'] / 1000) async def _snapshot(self, base: str, quote: str): url = self.rest_endpoints[0].route('l3book', self.sandbox).format(base, quote) timestamp = time() ret = await self.http_conn.read(url) await asyncio.sleep(1 / self.request_limit) ret = json.loads(ret, parse_float=Decimal) normalized = self.exchange_symbol_to_std_symbol(f"{base}-{quote}") self._l3_book[normalized] = OrderBook(self.id, normalized, max_depth=self.max_depth) for side, key in [(BID, 'BuyOrders'), (ASK, 'SellOrders')]: for order in ret[key]: price = Decimal(order['Price']) size = Decimal(order['Volume']) uuid = order['Guid'] self._order_ids[normalized][uuid] = (price, side) if price in self._l3_book[normalized].book[side]: self._l3_book[normalized].book[side][price][uuid] = size else: self._l3_book[normalized].book[side][price] = {uuid: size} await self.book_callback(L3_BOOK, self._l3_book[normalized], timestamp, raw=ret) async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if msg['Event'] == 'Trade': await self._trade(msg, timestamp) elif msg['Event'] in ('OrderCanceled', 'OrderChanged', 'NewOrder'): await self._book(msg, timestamp) elif msg['Event'] in ('Subscriptions', 'Heartbeat', 'Unsubscribe'): return else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() subs = [] for chan, symbols in conn.subscription.items(): if self.exchange_channel_to_std(chan) == L3_BOOK: subs.extend([chan.format(s) for s in set([sym.split("-")[0] for sym in symbols])]) else: subs.extend([chan.format(s) for s in symbols]) await conn.write(json.dumps({"Event": "Subscribe", "Data": subs}))
class BinanceFutures(Binance, BinanceFuturesRestMixin): id = BINANCE_FUTURES websocket_endpoints = [ WebsocketEndpoint('wss://fstream.binance.com', sandbox='wss://stream.binancefuture.com', options={'compression': None}) ] rest_endpoints = [ RestEndpoint('https://fapi.binance.com', sandbox='https://testnet.binancefuture.com', routes=Routes( '/fapi/v1/exchangeInfo', l2book='/fapi/v1/depth?symbol={}&limit={}', authentication='/fapi/v1/listenKey', open_interest='/fapi/v1//openInterest?symbol={}')) ] valid_depths = [5, 10, 20, 50, 100, 500, 1000] valid_depth_intervals = {'100ms', '250ms', '500ms'} websocket_channels = { **Binance.websocket_channels, FUNDING: 'markPrice', OPEN_INTEREST: 'open_interest', LIQUIDATIONS: 'forceOrder', POSITIONS: POSITIONS } @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: base, info = super()._parse_symbol_data(data) add = {} for symbol, orig in base.items(): if "_" in orig: continue add[f"{symbol.replace('PERP', 'PINDEX')}"] = f"p{orig}" base.update(add) return base, info def __init__(self, open_interest_interval=1.0, **kwargs): """ open_interest_interval: float time in seconds between open_interest polls """ super().__init__(**kwargs) self.open_interest_interval = open_interest_interval def _connect_rest(self): ret = [] for chan in set(self.subscription): if chan == 'open_interest': addrs = [ self.rest_endpoints[0].route( 'open_interest', sandbox=self.sandbox).format(pair) for pair in self.subscription[chan] ] ret.append((HTTPPoll(addrs, self.id, delay=60.0, sleep=self.open_interest_interval, proxy=self.http_proxy), self.subscribe, self.message_handler, self.authenticate)) return ret def _check_update_id(self, pair: str, msg: dict) -> bool: if self._l2_book[pair].delta is None and msg[ 'u'] < self.last_update_id[pair]: return True elif msg['U'] <= self.last_update_id[pair] <= msg['u']: self.last_update_id[pair] = msg['u'] return False elif self.last_update_id[pair] == msg['pu']: self.last_update_id[pair] = msg['u'] return False else: self._reset() LOG.warning("%s: Missing book update detected, resetting book", self.id) return True async def _open_interest(self, msg: dict, timestamp: float): """ { "openInterest": "10659.509", "symbol": "BTCUSDT", "time": 1589437530011 // Transaction time } """ pair = msg['symbol'] oi = msg['openInterest'] if oi != self._open_interest_cache.get(pair, None): o = OpenInterest(self.id, self.exchange_symbol_to_std_symbol(pair), Decimal(oi), self.timestamp_normalize(msg['time']), raw=msg) await self.callback(OPEN_INTEREST, o, timestamp) self._open_interest_cache[pair] = oi async def _account_update(self, msg: dict, timestamp: float): """ { "e": "ACCOUNT_UPDATE", // Event Type "E": 1564745798939, // Event Time "T": 1564745798938 , // Transaction "a": // Update Data { "m":"ORDER", // Event reason type "B":[ // Balances { "a":"USDT", // Asset "wb":"122624.12345678", // Wallet Balance "cw":"100.12345678", // Cross Wallet Balance "bc":"50.12345678" // Balance Change except PnL and Commission }, { "a":"BUSD", "wb":"1.00000000", "cw":"0.00000000", "bc":"-49.12345678" } ], "P":[ { "s":"BTCUSDT", // Symbol "pa":"0", // Position Amount "ep":"0.00000", // Entry Price "cr":"200", // (Pre-fee) Accumulated Realized "up":"0", // Unrealized PnL "mt":"isolated", // Margin Type "iw":"0.00000000", // Isolated Wallet (if isolated position) "ps":"BOTH" // Position Side }, { "s":"BTCUSDT", "pa":"20", "ep":"6563.66500", "cr":"0", "up":"2850.21200", "mt":"isolated", "iw":"13200.70726908", "ps":"LONG" }, { "s":"BTCUSDT", "pa":"-10", "ep":"6563.86000", "cr":"-45.04000000", "up":"-1423.15600", "mt":"isolated", "iw":"6570.42511771", "ps":"SHORT" } ] } } """ for balance in msg['a']['B']: b = Balance(self.id, balance['a'], Decimal(balance['wb']), None, raw=msg) await self.callback(BALANCES, b, timestamp) for position in msg['a']['P']: p = Position(self.id, self.exchange_symbol_to_std_symbol(position['s']), Decimal(position['pa']), Decimal(position['ep']), position['ps'].lower(), Decimal(position['up']), self.timestamp_normalize(msg['E']), raw=msg) await self.callback(POSITIONS, p, timestamp) async def _order_update(self, msg: dict, timestamp: float): """ { "e":"ORDER_TRADE_UPDATE", // Event Type "E":1568879465651, // Event Time "T":1568879465650, // Transaction Time "o": { "s":"BTCUSDT", // Symbol "c":"TEST", // Client Order Id // special client order id: // starts with "autoclose-": liquidation order // "adl_autoclose": ADL auto close order "S":"SELL", // Side "o":"TRAILING_STOP_MARKET", // Order Type "f":"GTC", // Time in Force "q":"0.001", // Original Quantity "p":"0", // Original Price "ap":"0", // Average Price "sp":"7103.04", // Stop Price. Please ignore with TRAILING_STOP_MARKET order "x":"NEW", // Execution Type "X":"NEW", // Order Status "i":8886774, // Order Id "l":"0", // Order Last Filled Quantity "z":"0", // Order Filled Accumulated Quantity "L":"0", // Last Filled Price "N":"USDT", // Commission Asset, will not push if no commission "n":"0", // Commission, will not push if no commission "T":1568879465651, // Order Trade Time "t":0, // Trade Id "b":"0", // Bids Notional "a":"9.91", // Ask Notional "m":false, // Is this trade the maker side? "R":false, // Is this reduce only "wt":"CONTRACT_PRICE", // Stop Price Working Type "ot":"TRAILING_STOP_MARKET", // Original Order Type "ps":"LONG", // Position Side "cp":false, // If Close-All, pushed with conditional order "AP":"7476.89", // Activation Price, only puhed with TRAILING_STOP_MARKET order "cr":"5.0", // Callback Rate, only puhed with TRAILING_STOP_MARKET order "rp":"0" // Realized Profit of the trade } } """ oi = OrderInfo(self.id, self.exchange_symbol_to_std_symbol(msg['o']['s']), str(msg['o']['i']), BUY if msg['o']['S'].lower() == 'buy' else SELL, msg['o']['x'], LIMIT if msg['o']['o'].lower() == 'limit' else MARKET if msg['o']['o'].lower() == 'market' else None, Decimal(msg['o']['ap']) if not Decimal.is_zero(Decimal(msg['o']['ap'])) else None, Decimal(msg['o']['q']), Decimal(msg['o']['q']) - Decimal(msg['o']['z']), self.timestamp_normalize(msg['E']), raw=msg) await self.callback(ORDER_INFO, oi, timestamp) async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float): msg = json.loads(msg, parse_float=Decimal) # Handle REST endpoint messages first if 'openInterest' in msg: return await self._open_interest(msg, timestamp) # Handle account updates from User Data Stream if self.requires_authentication: msg_type = msg.get('e') if msg_type == 'ACCOUNT_UPDATE': await self._account_update(msg, timestamp) elif msg_type == 'ORDER_TRADE_UPDATE': await self._order_update(msg, timestamp) return # Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>} # streamName is of format <symbol>@<channel> pair, _ = msg['stream'].split('@', 1) msg = msg['data'] pair = pair.upper() msg_type = msg.get('e') if msg_type == 'bookTicker': await self._ticker(msg, timestamp) elif msg_type == 'depthUpdate': await self._book(msg, pair, timestamp) elif msg_type == 'aggTrade': await self._trade(msg, timestamp) elif msg_type == 'forceOrder': await self._liquidations(msg, timestamp) elif msg_type == 'markPriceUpdate': await self._funding(msg, timestamp) elif msg['e'] == 'kline': await self._candle(msg, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class Bittrex(Feed): id = BITTREX websocket_endpoints = [ WebsocketEndpoint('wss://www.bitmex.com/realtime', authentication=True) ] rest_endpoints = [ RestEndpoint('https://api.bittrex.com', routes=Routes('/v3/markets', l2book='/v3/markets/{}/orderbook?depth={}')) ] valid_candle_intervals = {'1m', '5m', '1h', '1d'} valid_depths = [1, 25, 500] websocket_channels = { L2_BOOK: 'orderbook_{}_{}', TRADES: 'trade_{}', TICKER: 'ticker_{}', CANDLES: 'candle_{}_{}' } @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: info = {'instrument_type': {}} ret = {} for e in data: if e['status'] != 'ONLINE': continue s = Symbol(e['baseCurrencySymbol'], e['quoteCurrencySymbol']) ret[s.normalized] = e['symbol'] info['instrument_type'][s.normalized] = s.type return ret, info async def _ws_authentication(self, address: str, options: dict) -> Tuple[str, dict]: # Technically this isnt authentication, its the negotiation step for SignalR that # we are performing here since this method is called right before connecting r = self.http_sync.read( 'https://socket-v3.bittrex.com/signalr/negotiate', params={ 'connectionData': json.dumps([{ 'name': 'c3' }]), 'clientProtocol': 1.5 }, json=True) token = r['ConnectionToken'] url = requests.Request('GET', 'https://socket-v3.bittrex.com/signalr/connect', params={ 'transport': 'webSockets', 'connectionToken': token, 'connectionData': json.dumps([{ "name": "c3" }]), 'clientProtocol': 1.5 }).prepare().url return url.replace('https://', 'wss://'), options def __reset(self): self._l2_book = {} self.seq_no = {} def __depth(self): depth = self.valid_depths[-1] if self.max_depth: if 25 <= self.max_depth >= 500: depth = 500 else: depth = 25 return depth async def ticker(self, msg: dict, timestamp: float): """ { 'symbol': 'BTC-USDT', 'lastTradeRate': '38904.35254113', 'bidRate': '38868.52330647', 'askRate': '38886.38815323' } """ t = Ticker(self.id, self.exchange_symbol_to_std_symbol(msg['symbol']), Decimal(msg['bidRate']), Decimal(msg['askRate']), None, raw=msg) await self.callback(TICKER, t, timestamp) async def book(self, msg: dict, timestamp: float): """ { 'marketSymbol': 'BTC-USDT', 'depth': 500, 'sequence': 6032818, 'bidDeltas': [ { 'quantity': '0', 'rate': '38926.13088302' }, { 'quantity': '0.00213516', 'rate': '31881.73000000' } ], 'askDeltas': [ { 'quantity': '0.03106831', 'rate': '38989.50808432' }, { 'quantity': '0.27954874', 'rate': '39013.57939993' }, { 'quantity': '0', 'rate': '46667.67569819' } ] } """ pair = self.exchange_symbol_to_std_symbol(msg['marketSymbol']) seq_no = int(msg['sequence']) delta = {BID: [], ASK: []} if pair not in self._l2_book: await self._snapshot(pair, seq_no) else: if seq_no <= self.seq_no[pair]: return if seq_no != self.seq_no[pair] + 1: raise MissingSequenceNumber self.seq_no[pair] = seq_no for side, key in ((BID, 'bidDeltas'), (ASK, 'askDeltas')): for update in msg[key]: price = Decimal(update['rate']) size = Decimal(update['quantity']) if size == 0: delta[side].append((price, 0)) if price in self._l2_book[pair].book[side]: del self._l2_book[pair].book[side][price] else: self._l2_book[pair].book[side][price] = size delta[side].append((price, size)) await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg, sequence_number=seq_no, delta=delta) async def _snapshot(self, symbol: str, sequence_number: int): while True: ret, headers = await self.http_conn.read( self.rest_endpoints[0].route('l2book', self.sandbox).format( symbol, self.__depth()), return_headers=True) seq = int(headers['Sequence']) if seq >= sequence_number: break await asyncio.sleep(1.0) self.seq_no[symbol] = seq data = json.loads(ret, parse_float=Decimal) self._l2_book[symbol] = OrderBook(self.id, symbol, max_depth=self.max_depth) for side, entries in data.items(): self._l2_book[symbol].book[side] = { Decimal(e['rate']): Decimal(e['quantity']) for e in entries } await self.book_callback(L2_BOOK, self._l2_book[symbol], time.time(), raw=data, sequence_number=seq) async def trades(self, msg: dict, timestamp: float): """ { 'deltas': [ { 'id': '8e7f693b-6504-4cb7-9484-835435b147f9', 'executedAt': datetime.datetime(2021, 6, 13, 22, 38, 11, 80000, tzinfo=datetime.timezone.utc), 'quantity': '0.00693216', 'rate': '38808.83000000', 'takerSide': 'BUY' } ], 'sequence': 204392, 'marketSymbol': 'BTC-USD' } """ pair = self.exchange_symbol_to_std_symbol(msg['marketSymbol']) for trade in msg['deltas']: t = Trade(self.id, pair, BUY if trade['takerSide'] == 'BUY' else SELL, Decimal(trade['quantity']), Decimal(trade['rate']), self.timestamp_normalize(trade['executedAt']), id=trade['id'], raw=trade) await self.callback(TRADES, t, timestamp) async def candle(self, msg: dict, timestamp: float): """ { 'sequence': 134514, 'marketSymbol': 'BTC-USDT', 'interval': 'MINUTE_1', 'delta': { 'startsAt': datetime.datetime(2021, 6, 14, 1, 12, tzinfo=datetime.timezone.utc), 'open': '39023.31434847', 'high': '39023.31434847', 'low': '39023.31434847', 'close': '39023.31434847', 'volume': '0.05944473', 'quoteVolume': '2319.73038514' }, 'candleType': 'TRADE' } """ start = self.timestamp_normalize(msg['delta']['startsAt']) offset = 0 if self.candle_interval == '1m': offset = 60 elif self.candle_interval == '5m': offset = 300 elif self.candle_interval == '1h': offset = 3600 elif self.candle_interval == '1d': offset = 86400 end = start + offset c = Candle(self.id, self.exchange_symbol_to_std_symbol(msg['marketSymbol']), start, end, self.candle_interval, None, Decimal(msg['delta']['open']), Decimal(msg['delta']['close']), Decimal(msg['delta']['high']), Decimal(msg['delta']['low']), Decimal(msg['delta']['volume']), None, None, raw=msg) await self.callback(CANDLES, c, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg) if 'M' in msg and len(msg['M']) > 0: for update in msg['M']: if update['M'] == 'orderBook': for message in update['A']: data = json.loads(zlib.decompress( base64.b64decode(message), -zlib.MAX_WBITS).decode(), parse_float=Decimal) await self.book(data, timestamp) elif update['M'] == 'trade': for message in update['A']: data = json.loads(zlib.decompress( base64.b64decode(message), -zlib.MAX_WBITS).decode(), parse_float=Decimal) await self.trades(data, timestamp) elif update['M'] == 'ticker': for message in update['A']: data = json.loads(zlib.decompress( base64.b64decode(message), -zlib.MAX_WBITS).decode(), parse_float=Decimal) await self.ticker(data, timestamp) elif update['M'] == 'candle': for message in update['A']: data = json.loads(zlib.decompress( base64.b64decode(message), -zlib.MAX_WBITS).decode(), parse_float=Decimal) await self.candle(data, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) elif 'E' in msg: LOG.error("%s: Error from exchange %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() # H: Hub, M: Message, A: Args, I: Internal ID # For more signalR info see: # https://blog.3d-logic.com/2015/03/29/signalr-on-the-wire-an-informal-description-of-the-signalr-protocol/ # http://blogs.microsoft.co.il/applisec/2014/03/12/signalr-message-format/ for chan in self.subscription: channel = self.exchange_channel_to_std(chan) i = 1 for symbol in self.subscription[chan]: if channel == L2_BOOK: msg = { 'A': ([chan.format(symbol, self.__depth())], ), 'H': 'c3', 'I': i, 'M': 'Subscribe' } elif channel in (TRADES, TICKER): msg = { 'A': ([chan.format(symbol)], ), 'H': 'c3', 'I': i, 'M': 'Subscribe' } elif channel == CANDLES: interval = None if self.candle_interval == '1m': interval = 'MINUTE_1' elif self.candle_interval == '5m': interval = 'MINUTE_5' elif self.candle_interval == '1h': interval = 'HOUR_1' elif self.candle_interval == '1d': interval = 'DAY_1' msg = { 'A': ([chan.format(symbol, interval)], ), 'H': 'c3', 'I': i, 'M': 'Subscribe' } else: LOG.error("%s: invalid subscription for channel %s", channel) await conn.write(json.dumps(msg)) i += 1
class Bitget(Feed): id = BITGET websocket_endpoints = [ WebsocketEndpoint('wss://ws.bitget.com/spot/v1/stream') ] rest_endpoints = [ RestEndpoint('https://api.bitget.com', routes=Routes('/api/spot/v1/public/products')) ] valid_candle_intervals = { '1m', '5m', '15m', '30m', '1h', '4h', '12h', '1d', '1w' } websocket_channels = { L2_BOOK: 'books', TRADES: 'trade', TICKER: 'ticker', CANDLES: 'candle' } request_limit = 20 @classmethod def timestamp_normalize(cls, ts: int) -> float: return ts / 1000 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for entry in data['data']: """ { "baseCoin":"ALPHA", "makerFeeRate":"0.001", "maxTradeAmount":"0", "minTradeAmount":"2", "priceScale":"4", "quantityScale":"4", "quoteCoin":"USDT", "status":"online", "symbol":"ALPHAUSDT_SPBL", "symbolName":"ALPHAUSDT", "takerFeeRate":"0.001" } """ sym = Symbol(entry['baseCoin'], entry['quoteCoin']) info['instrument_type'][sym.normalized] = sym.type ret[sym.normalized] = entry['symbolName'] return ret, info def __reset(self): self._l2_book = {} async def _ticker(self, msg: dict, timestamp: float): """ { 'action': 'snapshot', 'arg': { 'instType': 'sp', 'channel': 'ticker', 'instId': 'BTCUSDT' }, 'data': [ { 'instId': 'BTCUSDT', 'last': '46572.07', 'open24h': '46414.54', 'high24h': '46767.30', 'low24h': '46221.11', 'bestBid': '46556.590000', 'bestAsk': '46565.670000', 'baseVolume': '1927.0855', 'quoteVolume': '89120317.8812', 'ts': 1649013100029, 'labeId': 0 } ] } """ for entry in msg['data']: # sometimes snapshots do not have bids/asks in them if 'bestBid' not in entry or 'bestAsk' not in entry: continue t = Ticker(self.id, self.exchange_symbol_to_std_symbol(entry['instId']), Decimal(entry['bestBid']), Decimal(entry['bestAsk']), self.timestamp_normalize(entry['ts']), raw=entry) await self.callback(TICKER, t, timestamp) async def _trade(self, msg: dict, timestamp: float): """ { 'action': 'update', 'arg': { 'instType': 'sp', 'channel': 'trade', 'instId': 'BTCUSDT' }, 'data': [ ['1649014224602', '46464.51', '0.0023', 'sell'] ] } """ for entry in msg['data']: t = Trade(self.id, self.exchange_symbol_to_std_symbol(msg['arg']['instId']), SELL if entry[3] == 'sell' else BUY, Decimal(entry[2]), Decimal(entry[1]), self.timestamp_normalize(int(entry[0])), raw=entry) await self.callback(TRADES, t, timestamp) async def _candle(self, msg: dict, timestamp: float): ''' { 'action': 'update', 'arg': { 'instType': 'sp', 'channel': 'candle1m', 'instId': 'BTCUSDT' }, 'data': [['1649014920000', '46434.2', '46437.98', '46434.2', '46437.98', '0.9469']] } ''' for entry in msg['data']: t = Candle(self.id, self.exchange_symbol_to_std_symbol( msg['arg']['instId']), self.timestamp_normalize(int(entry[0])), self.timestamp_normalize(int(entry[0])) + timedelta_str_to_sec(self.candle_interval), self.candle_interval, None, Decimal(entry[1]), Decimal(entry[4]), Decimal(entry[2]), Decimal(entry[3]), Decimal(entry[5]), None, self.timestamp_normalize(int(entry[0])), raw=entry) await self.callback(CANDLES, t, timestamp) async def _book(self, msg: dict, timestamp: float): sym = self.exchange_symbol_to_std_symbol(msg['arg']['instId']) data = msg['data'][0] if msg['action'] == 'snapshot': ''' { 'action': 'snapshot', 'arg': { 'instType': 'sp', 'channel': 'books', 'instId': 'BTCUSDT' }, 'data': [ { 'asks': [['46700.38', '0.0554'], ['46701.25', '0.0147'], ... 'bids': [['46686.68', '0.0032'], ['46684.75', '0.0161'], ... 'checksum': -393656186, 'ts': '1649021358917' } ] } ''' bids = { Decimal(price): Decimal(amount) for price, amount in data['bids'] } asks = { Decimal(price): Decimal(amount) for price, amount in data['asks'] } self._l2_book[sym] = OrderBook(self.id, sym, max_depth=self.max_depth, bids=bids, asks=asks) await self.book_callback(L2_BOOK, self._l2_book[sym], timestamp, checksum=data['checksum'], timestamp=self.timestamp_normalize( int(data['ts'])), raw=msg) else: ''' { 'action': 'update', 'arg': { 'instType': 'sp', 'channel': 'books', 'instId': 'BTCUSDT' }, 'data': [ { 'asks': [['46701.25', '0'], ['46701.46', '0.0054'], ... 'bids': [['46687.67', '0.0531'], ['46686.22', '0'], ... 'checksum': -750266015, 'ts': '1649021359467' } ] } ''' delta = {BID: [], ASK: []} for side, key in ((BID, 'bids'), (ASK, 'asks')): for price, size in data[key]: price = Decimal(price) size = Decimal(size) delta[side].append((price, size)) if size == 0: del self._l2_book[sym].book[side][price] else: self._l2_book[sym].book[side][price] = size await self.book_callback(L2_BOOK, self._l2_book[sym], timestamp, delta=delta, checksum=data['checksum'], timestamp=self.timestamp_normalize( int(data['ts'])), raw=msg) async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'event' in msg: # {'event': 'subscribe', 'arg': {'instType': 'sp', 'channel': 'ticker', 'instId': 'BTCUSDT'}} if msg['event'] == 'subscribe': return if msg['event'] == 'error': LOG.error('%s: Error from exchange: %s', conn.uuid, msg) return if msg['arg']['channel'] == 'books': await self._book(msg, timestamp) elif msg['arg']['channel'] == 'ticker': await self._ticker(msg, timestamp) elif msg['arg']['channel'] == 'trade': await self._trade(msg, timestamp) elif msg['arg']['channel'].startswith('candle'): await self._candle(msg, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() args = [] interval = self.candle_interval if interval[-1] != 'm': interval[-1] = interval[-1].upper() for chan, symbols in conn.subscription.items(): for s in symbols: d = { 'instType': 'SPBL' if self.is_authenticated_channel( self.exchange_channel_to_std(chan)) else 'SP', 'channel': chan if chan != 'candle' else 'candle' + interval, 'instId': s } args.append(d) await conn.write(json.dumps({"op": "subscribe", "args": args}))
class BinanceDelivery(Binance, BinanceDeliveryRestMixin): id = BINANCE_DELIVERY websocket_endpoints = [ WebsocketEndpoint('wss://dstream.binance.com', options={'compression': None}) ] rest_endpoints = [ RestEndpoint('https://dapi.binance.com', routes=Routes('/dapi/v1/exchangeInfo', l2book='/dapi/v1/depth?symbol={}&limit={}', authentication='/dapi/v1/listenKey')) ] valid_depths = [5, 10, 20, 50, 100, 500, 1000] valid_depth_intervals = {'100ms', '250ms', '500ms'} websocket_channels = { **Binance.websocket_channels, FUNDING: 'markPrice', OPEN_INTEREST: 'open_interest', LIQUIDATIONS: 'forceOrder', POSITIONS: POSITIONS } def _check_update_id(self, pair: str, msg: dict) -> bool: if self._l2_book[pair].delta is None and msg[ 'u'] < self.last_update_id[pair]: return True elif msg['U'] <= self.last_update_id[pair] <= msg['u']: self.last_update_id[pair] = msg['u'] return False elif self.last_update_id[pair] == msg['pu']: self.last_update_id[pair] = msg['u'] return False else: self._reset() LOG.warning("%s: Missing book update detected, resetting book", self.id) return True async def _account_update(self, msg: dict, timestamp: float): """ { "e": "ACCOUNT_UPDATE", // Event Type "E": 1564745798939, // Event Time "T": 1564745798938 , // Transaction "i": "SfsR", // Account Alias "a": // Update Data { "m":"ORDER", // Event reason type "B":[ // Balances { "a":"BTC", // Asset "wb":"122624.12345678", // Wallet Balance "cw":"100.12345678" // Cross Wallet Balance }, { "a":"ETH", "wb":"1.00000000", "cw":"0.00000000" } ], "P":[ { "s":"BTCUSD_200925", // Symbol "pa":"0", // Position Amount "ep":"0.0", // Entry Price "cr":"200", // (Pre-fee) Accumulated Realized "up":"0", // Unrealized PnL "mt":"isolated", // Margin Type "iw":"0.00000000", // Isolated Wallet (if isolated position) "ps":"BOTH" // Position Side }, { "s":"BTCUSD_200925", "pa":"20", "ep":"6563.6", "cr":"0", "up":"2850.21200000", "mt":"isolated", "iw":"13200.70726908", "ps":"LONG" }, { "s":"BTCUSD_200925", "pa":"-10", "ep":"6563.8", "cr":"-45.04000000", "up":"-1423.15600000", "mt":"isolated", "iw":"6570.42511771", "ps":"SHORT" } ] } } """ for balance in msg['a']['B']: b = Balance(self.id, balance['a'], Decimal(balance['wb']), None, raw=msg) await self.callback(BALANCES, b, timestamp) for position in msg['a']['P']: p = Position(self.id, self.exchange_symbol_to_std_symbol(position['s']), Decimal(position['pa']), Decimal(position['ep']), position['ps'].lower(), Decimal(position['up']), self.timestamp_normalize(msg['E']), raw=msg) await self.callback(POSITIONS, p, timestamp) async def _order_update(self, msg: dict, timestamp: float): """ { "e":"ORDER_TRADE_UPDATE", // Event Type "E":1591274595442, // Event Time "T":1591274595453, // Transaction Time "i":"SfsR", // Account Alias "o": { "s":"BTCUSD_200925", // Symbol "c":"TEST", // Client Order Id // special client order id: // starts with "autoclose-": liquidation order // "adl_autoclose": ADL auto close order "S":"SELL", // Side "o":"TRAILING_STOP_MARKET", // Order Type "f":"GTC", // Time in Force "q":"2", // Original Quantity "p":"0", // Original Price "ap":"0", // Average Price "sp":"9103.1", // Stop Price. Please ignore with TRAILING_STOP_MARKET order "x":"NEW", // Execution Type "X":"NEW", // Order Status "i":8888888, // Order Id "l":"0", // Order Last Filled Quantity "z":"0", // Order Filled Accumulated Quantity "L":"0", // Last Filled Price "ma": "BTC", // Margin Asset "N":"BTC", // Commission Asset of the trade, will not push if no commission "n":"0", // Commission of the trade, will not push if no commission "T":1591274595442, // Order Trade Time "t":0, // Trade Id "rp": "0", // Realized Profit of the trade "b":"0", // Bid quantity of base asset "a":"0", // Ask quantity of base asset "m":false, // Is this trade the maker side? "R":false, // Is this reduce only "wt":"CONTRACT_PRICE", // Stop Price Working Type "ot":"TRAILING_STOP_MARKET",// Original Order Type "ps":"LONG", // Position Side "cp":false, // If Close-All, pushed with conditional order "AP":"9476.8", // Activation Price, only puhed with TRAILING_STOP_MARKET order "cr":"5.0", // Callback Rate, only puhed with TRAILING_STOP_MARKET order "pP": false // If conditional order trigger is protected } } """ oi = OrderInfo(self.id, self.exchange_symbol_to_std_symbol(msg['o']['s']), str(msg['o']['i']), BUY if msg['o']['S'].lower() == 'buy' else SELL, msg['o']['x'], LIMIT if msg['o']['o'].lower() == 'limit' else MARKET if msg['o']['o'].lower() == 'market' else None, Decimal(msg['o']['ap']) if not Decimal.is_zero(Decimal(msg['o']['ap'])) else None, Decimal(msg['o']['q']), Decimal(msg['o']['q']) - Decimal(msg['o']['z']), self.timestamp_normalize(msg['E']), raw=msg) await self.callback(ORDER_INFO, oi, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) # Handle account updates from User Data Stream if self.requires_authentication: msg_type = msg.get('e') if msg_type == 'ACCOUNT_UPDATE': await self._account_update(msg, timestamp) elif msg_type == 'ORDER_TRADE_UPDATE': await self._order_update(msg, timestamp) return # Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>} # streamName is of format <symbol>@<channel> pair, _ = msg['stream'].split('@', 1) msg = msg['data'] pair = pair.upper() msg_type = msg.get('e') if msg_type == 'bookTicker': await self._ticker(msg, timestamp) elif msg_type == 'depthUpdate': await self._book(msg, pair, timestamp) elif msg_type == 'aggTrade': await self._trade(msg, timestamp) elif msg_type == 'forceOrder': await self._liquidations(msg, timestamp) elif msg_type == 'markPriceUpdate': await self._funding(msg, timestamp) elif msg_type == 'kline': await self._candle(msg, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class OKCoin(Feed): id = OKCOIN websocket_endpoints = [WebsocketEndpoint('wss://real.okcoin.com:8443/ws/v3')] rest_endpoints = [RestEndpoint('https://www.okcoin.com', routes=Routes('/api/spot/v3/instruments'))] websocket_channels = { L2_BOOK: 'spot/depth_l2_tbt', TRADES: 'spot/trade', TICKER: 'spot/ticker', } @classmethod def timestamp_normalize(cls, ts) -> float: return ts.timestamp() @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for e in data: s = Symbol(e['base_currency'], e['quote_currency']) ret[s.normalized] = e['instrument_id'] info['tick_size'][s.normalized] = e['tick_size'] info['instrument_type'][s.normalized] = s.type return ret, info def __reset(self): self._l2_book = {} async def subscribe(self, conn: AsyncConnection): self.__reset() for chan in self.subscription: for symbol in self.subscription[chan]: request = {"op": "subscribe", "args": f"{chan}:{symbol}"} await conn.write(json.dumps(request)) async def _ticker(self, msg: dict, timestamp: float): """ { 'table': 'spot/ticker', 'data': [ { 'instrument_id': 'BTC-USD', 'last': '3977.74', 'best_bid': '3977.08', 'best_ask': '3978.73', 'open_24h': '3978.21', 'high_24h': '3995.43', 'low_24h': '3961.02', 'base_volume_24h': '248.245', 'quote_volume_24h': '988112.225861', 'timestamp': '2019-03-22T22:26:34.019Z' } ] } """ for update in msg['data']: pair = update['instrument_id'] update_timestamp = self.timestamp_normalize(update['timestamp']) t = Ticker(self.id, pair, Decimal(update['best_bid']) if update['best_bid'] else Decimal(0), Decimal(update['best_ask']) if update['best_ask'] else Decimal(0), update_timestamp, raw=update) await self.callback(TICKER, t, timestamp) async def _trade(self, msg: dict, timestamp: float): """ {'table': 'spot/trade', 'data': [{'instrument_id': 'BTC-USD', 'price': '3977.44', 'side': 'buy', 'size': '0.0096', 'timestamp': '2019-03-22T22:45:44.578Z', 'trade_id': '486519521'}]} """ for trade in msg['data']: t = Trade( self.id, self.exchange_symbol_to_std_symbol(trade['instrument_id']), BUY if trade['side'] == 'buy' else SELL, Decimal(trade['size']), Decimal(trade['price']), self.timestamp_normalize(trade['timestamp']), id=trade['trade_id'], raw=trade ) await self.callback(TRADES, t, timestamp) async def _book(self, msg: dict, timestamp: float): if msg['action'] == 'partial': # snapshot for update in msg['data']: pair = self.exchange_symbol_to_std_symbol(update['instrument_id']) bids = {Decimal(price): Decimal(amount) for price, amount, *_ in update['bids']} asks = {Decimal(price): Decimal(amount) for price, amount, *_ in update['asks']} self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, checksum_format='OKCOIN', bids=bids, asks=asks) if self.checksum_validation and self._l2_book[pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF): raise BadChecksum await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(update['timestamp']), raw=msg, checksum=update['checksum'] & 0xFFFFFFFF) else: # update for update in msg['data']: delta = {BID: [], ASK: []} pair = self.exchange_symbol_to_std_symbol(update['instrument_id']) for side in ('bids', 'asks'): s = BID if side == 'bids' else ASK for price, amount, *_ in update[side]: price = Decimal(price) amount = Decimal(amount) if amount == 0: if price in self._l2_book[pair].book[s]: delta[s].append((price, 0)) del self._l2_book[pair].book[s][price] else: delta[s].append((price, amount)) self._l2_book[pair].book[s][price] = amount if self.checksum_validation and self._l2_book[pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF): raise BadChecksum await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(update['timestamp']), raw=msg, delta=delta, checksum=update['checksum'] & 0xFFFFFFFF) async def message_handler(self, msg: str, conn, timestamp: float): # DEFLATE compression, no header msg = zlib.decompress(msg, -15) msg = json.loads(msg, parse_float=Decimal) if 'event' in msg: if msg['event'] == 'error': LOG.error("%s: Error: %s", self.id, msg) elif msg['event'] == 'subscribe': pass else: LOG.warning("%s: Unhandled event %s", self.id, msg) elif 'table' in msg: if 'ticker' in msg['table']: await self._ticker(msg, timestamp) elif 'trade' in msg['table']: await self._trade(msg, timestamp) elif 'depth_l2_tbt' in msg['table']: await self._book(msg, timestamp) elif 'spot/order' in msg['table']: await self._order(msg, timestamp) else: LOG.warning("%s: Unhandled message %s", self.id, msg) else: LOG.warning("%s: Unhandled message %s", self.id, msg)
class BitDotCom(Feed): id = BITDOTCOM websocket_endpoints = [ WebsocketEndpoint('wss://spot-ws.bit.com', instrument_filter=('TYPE', (SPOT, )), sandbox='wss://betaspot-ws.bitexch.dev'), WebsocketEndpoint('wss://ws.bit.com', instrument_filter=('TYPE', (FUTURES, OPTION, PERPETUAL)), sandbox='wss://betaws.bitexch.dev'), ] rest_endpoints = [ RestEndpoint('https://spot-api.bit.com', instrument_filter=('TYPE', (SPOT, )), sandbox='https://betaspot-api.bitexch.dev', routes=Routes('/spot/v1/instruments', authentication='/spot/v1/ws/auth')), RestEndpoint('https://api.bit.com', instrument_filter=('TYPE', (OPTION, FUTURES, PERPETUAL)), sandbox='https://betaapi.bitexch.dev', routes=Routes('/v1/instruments?currency={}&active=true', currencies='/v1/currencies', authentication='/v1/ws/auth')) ] websocket_channels = { L2_BOOK: 'depth', TRADES: 'trade', TICKER: 'ticker', ORDER_INFO: 'order', BALANCES: 'account', FILLS: 'user_trade', # funding rates paid and received } request_limit = 10 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._sequence_no = defaultdict(int) @classmethod def _symbol_endpoint_prepare(cls, ep: RestEndpoint) -> Union[List[str], str]: if ep.routes.currencies: ret = cls.http_sync.read(ep.route('currencies'), json=True, uuid=cls.id) return [ ep.route('instruments').format(currency) for currency in ret['data']['currencies'] ] return ep.route('instruments') @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 @classmethod def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for entry in data: if entry['code'] != 0: raise ValueError('%s - Failed to collect instrument data - %s', cls.id, entry['message']) for mapping in entry['data']: if 'category' in mapping: expiry = None strike = None otype = None if mapping['category'] == 'option': stype = OPTION strike = int(float(mapping['strike_price'])) expiry = cls.timestamp_normalize( mapping['expiration_at']) otype = mapping['option_type'] elif mapping['category'] == 'future': if 'PERPETUAL' in mapping['instrument_id']: stype = PERPETUAL else: stype = FUTURES expiry = cls.timestamp_normalize( mapping['expiration_at']) s = Symbol(mapping['base_currency'], mapping['quote_currency'], type=stype, option_type=otype, expiry_date=expiry, strike_price=strike) ret[s.normalized] = mapping['instrument_id'] info['instrument_type'][s.normalized] = stype else: # Spot s = Symbol(mapping['base_currency'], mapping['quote_currency'], type=SPOT) ret[s.normalized] = mapping['pair'] info['instrument_type'][s.normalized] = SPOT return ret, info def __reset(self, conn: AsyncConnection): if self.std_channel_to_exchange(L2_BOOK) in conn.subscription: for pair in conn.subscription[self.std_channel_to_exchange( L2_BOOK)]: std_pair = self.exchange_symbol_to_std_symbol(pair) if std_pair in self._l2_book: del self._l2_book[std_pair] if std_pair in self._sequence_no: del self._sequence_no[std_pair] def encode_list(self, item_list: list): list_val = [] for item in item_list: obj_val = self.encode_object(item) list_val.append(obj_val) output = '&'.join(list_val) return '[' + output + ']' def get_signature(self, api_path: str, param_map: dict): str_to_sign = api_path + '&' + self.encode_object(param_map) return hmac.new(self.key_secret.encode('utf-8'), str_to_sign.encode('utf-8'), digestmod=hashlib.sha256).hexdigest() def encode_object(self, param_map: dict): sorted_keys = sorted(param_map.keys()) ret_list = [] for key in sorted_keys: val = param_map[key] if isinstance(val, list): list_val = self.encode_list(val) ret_list.append(f'{key}={list_val}') elif isinstance(val, dict): dict_val = self.encode_object(val) ret_list.append(f'{key}={dict_val}') elif isinstance(val, bool): bool_val = str(val).lower() ret_list.append(f'{key}={bool_val}') else: general_val = str(val) ret_list.append(f'{key}={general_val}') sorted_list = sorted(ret_list) return '&'.join(sorted_list) async def authenticate(self, connection: AsyncConnection): if not self.key_id or not self.key_secret: return if any([ self.is_authenticated_channel(self.exchange_channel_to_std(c)) for c in connection.subscription ]): symbols = list( set(itertools.chain(*connection.subscription.values()))) sym = str_to_symbol(self.exchange_symbol_to_std_symbol(symbols[0])) for ep in self.rest_endpoints: if sym.type in ep.instrument_filter[1]: ts = int(round(time.time() * 1000)) signature = self.get_signature(ep.routes.authentication, {'timestamp': ts}) params = {'timestamp': ts, 'signature': signature} ret = self.http_sync.read( ep.route('authentication', sandbox=self.sandbox), params=params, headers={'X-Bit-Access-Key': self.key_id}, json=True) if ret['code'] != 0 or 'token' not in ret['data']: LOG.warning('%s: authentication failed: %s', ret) token = ret['data']['token'] self._auth_token = token return async def subscribe(self, connection: AsyncConnection): self.__reset(connection) for chan, symbols in connection.subscription.items(): if len(symbols) == 0: continue stype = str_to_symbol( self.exchange_symbol_to_std_symbol(symbols[0])).type msg = { 'type': 'subscribe', 'channels': [chan], 'instruments' if stype in {PERPETUAL, FUTURES, OPTION} else 'pairs': symbols, } if self.is_authenticated_channel( self.exchange_channel_to_std(chan)): msg['token'] = self._auth_token await connection.write(json.dumps(msg)) async def _trade(self, data: dict, timestamp: float): """ { 'channel': 'trade', 'timestamp': 1639080717242, 'data': [{ 'trade_id': '7016884324', 'instrument_id': 'BTC-PERPETUAL', 'price': '47482.50000000', 'qty': '6000.00000000', 'side': 'sell', 'sigma': '0.00000000', 'is_block_trade': False, 'created_at': 1639080717195 }] } """ for t in data['data']: trade = Trade(self.id, self.exchange_symbol_to_std_symbol( t.get('instrument_id') or t.get('pair')), SELL if t['side'] == 'sell' else BUY, Decimal(t['qty']), Decimal(t['price']), self.timestamp_normalize(t['created_at']), id=t['trade_id'], raw=t) await self.callback(TRADES, trade, timestamp) async def _book(self, data: dict, timestamp: float): ''' Snapshot { 'channel': 'depth', 'timestamp': 1639083660346, 'data': { 'type': 'snapshot', 'instrument_id': 'BTC-PERPETUAL', 'sequence': 1639042602148589825, 'bids': [ ['47763.00000000', '20000.00000000'], ['47762.50000000', '6260.00000000'], ... ] 'asks': [ ['47768.00000000', '10000.00000000'], ['47776.50000000', '20000.00000000'], ... ] } } Delta { 'channel': 'depth', 'timestamp': 1639083660401, 'data': { 'type': 'update', 'instrument_id': 'BTC-PERPETUAL', 'sequence': 1639042602148589842, 'prev_sequence': 1639042602148589841, 'changes': [ ['sell', '47874.00000000', '0.00000000'] ] } } ''' if data['data']['type'] == 'update': pair = self.exchange_symbol_to_std_symbol( data['data'].get('instrument_id') or data['data'].get('pair')) if data['data']['sequence'] != self._sequence_no[pair] + 1: raise MissingSequenceNumber( "Missing sequence number, restarting") self._sequence_no[pair] = data['data']['sequence'] delta = {BID: [], ASK: []} for side, price, amount in data['data']['changes']: side = ASK if side == 'sell' else BID price = Decimal(price) amount = Decimal(amount) if amount == 0: delta[side].append((price, 0)) del self._l2_book[pair].book[side][price] else: delta[side].append((price, amount)) self._l2_book[pair].book[side][price] = amount await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize( data['timestamp']), raw=data, sequence_number=self._sequence_no[pair], delta=delta) else: pair = self.exchange_symbol_to_std_symbol( data['data'].get('instrument_id') or data['data'].get('pair')) self._l2_book[pair] = OrderBook( self.id, pair, max_depth=self.max_depth, bids={ Decimal(price): Decimal(size) for price, size in data['data']['bids'] }, asks={ Decimal(price): Decimal(size) for price, size in data['data']['asks'] }) self._sequence_no[pair] = data['data']['sequence'] await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize( data['timestamp']), raw=data, sequence_number=data['data']['sequence']) async def _ticker(self, data: dict, timestamp: float): ''' { 'channel': 'ticker', 'timestamp': 1639093870710, 'data': { 'time': 1639093870710, 'instrument_id': 'ETH-PERPETUAL', 'best_bid': '4155.85000000', 'best_ask': '4155.90000000', 'best_bid_qty': '2000.00000000', 'best_ask_qty': '3000.00000000', 'ask_sigma': '', 'bid_sigma': '', 'last_price': '4157.80000000', 'last_qty': '1000.00000000', 'open24h': '4436.75000000', 'high24h': '4490.00000000', 'low24h': '4086.60000000', 'price_change24h': '-0.06287260', 'volume24h': '1000218.00000000', 'open_interest': '7564685.00000000', 'funding_rate': '0.00025108', 'funding_rate8h': '0.00006396', 'mark_price': '4155.62874869', 'min_sell': '4030.50000000', 'max_buy': '4280.50000000' } } ''' if data['data']['best_bid'] and data['data']['best_ask']: t = Ticker(self.id, self.exchange_symbol_to_std_symbol( data['data'].get('instrument_id') or data['data'].get('pair')), Decimal(data['data']['best_bid']), Decimal(data['data']['best_ask']), self.timestamp_normalize(data['timestamp']), raw=data) await self.callback(TICKER, t, timestamp) def _order_type_translate(self, t: str) -> str: if t == 'limit': return LIMIT if t == 'market': return MARKET if t == 'stop-limit': return STOP_LIMIT if t == 'stop-market': return STOP_MARKET if t == 'trigger-limit': return TRIGGER_LIMIT if t == 'trigger-market': return TRIGGER_MARKET raise ValueError('Invalid order type detected %s', t) def _status_translate(self, s: str) -> str: if s == 'open': return OPEN if s == 'pending': return PENDING if s == 'filled': return FILLED if s == 'cancelled': return CANCELLED raise ValueError('Invalid order status detected %s', s) async def _order(self, msg: dict, timestamp: float): """ { "channel":"order", "timestamp":1587994934089, "data":[ { "order_id":"1590", "instrument_id":"BTC-1MAY20-8750-P", "qty":"0.50000000", "filled_qty":"0.10000000", "remain_qty":"0.40000000", "price":"0.16000000", "avg_price":"0.16000000", "side":"buy", "order_type":"limit", "time_in_force":"gtc", "created_at":1587870609000, "updated_at":1587870609000, "status":"open", "fee":"0.00002000", "cash_flow":"-0.01600000", "pnl":"0.00000000", "is_liquidation": false, "auto_price":"0.00000000", "auto_price_type":"", "taker_fee_rate": "0.00050000", "maker_fee_rate": "0.00020000", "label": "hedge", "stop_price": "0.00000000", "reduce_only": false, "post_only": false, "reject_post_only": false, "mmp": false, "reorder_index": 1 } ] } """ for entry in msg['data']: oi = OrderInfo(self.id, self.exchange_symbol_to_std_symbol( entry['instrument_id']), entry['order_id'], BUY if entry['side'] == 'buy' else SELL, self._status_translate(entry['status']), self._order_type_translate(entry['order_type']), Decimal(entry['price']), Decimal(entry['filled_qty']), Decimal(entry['remain_qty']), self.timestamp_normalize(entry['updated_at']), raw=entry) await self.callback(ORDER_INFO, oi, timestamp) async def _balances(self, msg: dict, timestamp: float): ''' Futures/Options { "channel":"account", "timestamp":1589031930115, "data":{ "user_id":"53345", "currency":"BTC", "cash_balance":"9999.94981346", "available_balance":"9999.90496213", "margin_balance":"9999.94981421", "initial_margin":"0.04485208", "maintenance_margin":"0.00000114", "equity":"10000.00074364", "pnl":"0.00746583", "total_delta":"0.06207078", "account_id":"8", "mode":"regular", "session_upl":"0.00081244", "session_rpl":"0.00000021", "option_value":"0.05092943", "option_pnl":"0.00737943", "option_session_rpl":"0.00000000", "option_session_upl":"0.00081190", "option_delta":"0.11279249", "option_gamma":"0.00002905", "option_vega":"4.30272923", "option_theta":"-3.08908220", "future_pnl":"0.00008640", "future_session_rpl":"0.00000021", "future_session_upl":"0.00000054", "future_session_funding":"0.00000021", "future_delta":"0.00955630", "created_at":1588997840512, "projected_info": { "projected_initial_margin": "0.97919888", "projected_maintenance_margin": "0.78335911", "projected_total_delta": "3.89635553" } } } Spot { 'channel': 'account', 'timestamp': 1641516119102, 'data': { 'user_id': '979394', 'balances': [ { 'currency': 'BTC', 'available': '31.02527500', 'frozen': '0.00000000' }, { 'currency': 'ETH', 'available': '110.00000000', 'frozen': '0.00000000' } ] } } ''' if 'balances' in msg['data']: # Spot for balance in msg['data']['balances']: b = Balance(self.id, balance['currency'], Decimal(balance['available']), Decimal(balance['frozen']), raw=msg) await self.callback(BALANCES, b, timestamp) else: b = Balance(self.id, msg['data']['currency'], Decimal(msg['data']['cash_balance']), Decimal(msg['data']['cash_balance']) - Decimal(msg['data']['available_balance']), raw=msg) await self.callback(BALANCES, b, timestamp) async def _fill(self, msg: dict, timestamp: float): ''' { "channel":"user_trade", "timestamp":1588997059737, "data":[ { "trade_id":2388418, "order_id":1384232, "instrument_id":"BTC-26JUN20-6000-P", "qty":"0.10000000", "price":"0.01800000", "sigma":"1.15054346", "underlying_price":"9905.54000000", "index_price":"9850.47000000", "usd_price":"177.30846000", "fee":"0.00005000", "fee_rate":"0.00050000", "side":"buy", "created_at":1588997060000, "is_taker":true, "order_type":"limit", "is_block_trade":false, "label": "hedge" } ] } ''' for entry in msg['data']: f = Fill(self.id, self.exchange_symbol_to_std_symbol( entry['instrument_id']), BUY if entry['side'] == 'buy' else SELL, Decimal(entry['qty']), Decimal(entry['price']), Decimal(entry['fee']), str(entry['trade_id']), str(entry['order_id']), self._order_type_translate(entry['order_type']), None, self.timestamp_normalize(entry['created_at']), raw=entry) await self.callback(FILLS, f, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if msg['channel'] == 'depth': await self._book(msg, timestamp) elif msg['channel'] == 'trade': await self._trade(msg, timestamp) elif msg['channel'] == 'ticker': await self._ticker(msg, timestamp) elif msg['channel'] == 'order': await self._order(msg, timestamp) elif msg['channel'] == 'account': await self._balances(msg, timestamp) elif msg['channel'] == 'user_trade': await self._fill(msg, timestamp) elif msg['channel'] == 'subscription': """ { 'channel': 'subscription', 'timestamp': 1639080572093, 'data': {'code': 0, 'subscription': ['trade']} } """ if msg['data']['code'] == 0: return else: LOG.warning( "%s: error received from exchange while subscribing: %s", self.id, msg) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class Coinbase(Feed, CoinbaseRestMixin): id = COINBASE websocket_endpoints = [WebsocketEndpoint('wss://ws-feed.pro.coinbase.com', options={'compression': None})] rest_endpoints = [RestEndpoint('https://api.pro.coinbase.com', routes=Routes('/products', l3book='/products/{}/book?level=3'))] websocket_channels = { L2_BOOK: 'level2', L3_BOOK: 'full', TRADES: 'matches', TICKER: 'ticker', } request_limit = 10 @classmethod def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for entry in data: sym = Symbol(entry['base_currency'], entry['quote_currency']) info['tick_size'][sym.normalized] = entry['quote_increment'] info['instrument_type'][sym.normalized] = sym.type ret[sym.normalized] = entry['id'] return ret, info def __init__(self, callbacks=None, **kwargs): super().__init__(callbacks=callbacks, **kwargs) # we only keep track of the L3 order book if we have at least one subscribed order-book callback. # use case: subscribing to the L3 book plus Trade type gives you order_type information (see _received below), # and we don't need to do the rest of the book-keeping unless we have an active callback self.keep_l3_book = False if callbacks and L3_BOOK in callbacks: self.keep_l3_book = True self.__reset() def __reset(self): self.order_map = {} self.order_type_map = {} self.seq_no = None # sequence number validation only works when the FULL data stream is enabled chan = self.std_channel_to_exchange(L3_BOOK) if chan in self.subscription: pairs = self.subscription[chan] self.seq_no = {pair: None for pair in pairs} self._l3_book = {} self._l2_book = {} async def _ticker(self, msg: dict, timestamp: float): ''' { 'type': 'ticker', 'sequence': 5928281084, 'product_id': 'BTC-USD', 'price': '8500.01000000', 'open_24h': '8217.24000000', 'volume_24h': '4529.1293778', 'low_24h': '8172.00000000', 'high_24h': '8600.00000000', 'volume_30d': '329178.93594133', 'best_bid': '8500', 'best_ask': '8500.01' } { 'type': 'ticker', 'sequence': 5928281348, 'product_id': 'BTC-USD', 'price': '8500.00000000', 'open_24h': '8217.24000000', 'volume_24h': '4529.13179472', 'low_24h': '8172.00000000', 'high_24h': '8600.00000000', 'volume_30d': '329178.93835825', 'best_bid': '8500', 'best_ask': '8500.01', 'side': 'sell', 'time': '2018-05-21T00:30:11.587000Z', 'trade_id': 43736677, 'last_size': '0.00241692' } ''' await self.callback(TICKER, Ticker(self.id, self.exchange_symbol_to_std_symbol(msg['product_id']), Decimal(msg['best_bid']), Decimal(msg['best_ask']), self.timestamp_normalize(msg['time']), raw=msg), timestamp) async def _book_update(self, msg: dict, timestamp: float): ''' { 'type': 'match', or last_match 'trade_id': 43736593 'maker_order_id': '2663b65f-b74e-4513-909d-975e3910cf22', 'taker_order_id': 'd058d737-87f1-4763-bbb4-c2ccf2a40bde', 'side': 'buy', 'size': '0.01235647', 'price': '8506.26000000', 'product_id': 'BTC-USD', 'sequence': 5928276661, 'time': '2018-05-21T00:26:05.585000Z' } ''' pair = self.exchange_symbol_to_std_symbol(msg['product_id']) ts = self.timestamp_normalize(msg['time']) if self.keep_l3_book and 'full' in self.subscription and pair in self.subscription['full']: delta = {BID: [], ASK: []} price = Decimal(msg['price']) side = ASK if msg['side'] == 'sell' else BID size = Decimal(msg['size']) maker_order_id = msg['maker_order_id'] _, new_size = self.order_map[maker_order_id] new_size -= size if new_size <= 0: del self.order_map[maker_order_id] self.order_type_map.pop(maker_order_id, None) delta[side].append((maker_order_id, price, 0)) del self._l3_book[pair].book[side][price][maker_order_id] if len(self._l3_book[pair].book[side][price]) == 0: del self._l3_book[pair].book[side][price] else: self.order_map[maker_order_id] = (price, new_size) self._l3_book[pair].book[side][price][maker_order_id] = new_size delta[side].append((maker_order_id, price, new_size)) await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=ts, delta=delta, raw=msg, sequence_number=self.seq_no[pair]) order_type = self.order_type_map.get(msg['taker_order_id']) t = Trade( self.id, self.exchange_symbol_to_std_symbol(msg['product_id']), SELL if msg['side'] == 'buy' else BUY, Decimal(msg['size']), Decimal(msg['price']), ts, id=str(msg['trade_id']), type=order_type, raw=msg ) await self.callback(TRADES, t, timestamp) async def _pair_level2_snapshot(self, msg: dict, timestamp: float): pair = self.exchange_symbol_to_std_symbol(msg['product_id']) bids = {Decimal(price): Decimal(amount) for price, amount in msg['bids']} asks = {Decimal(price): Decimal(amount) for price, amount in msg['asks']} if pair not in self._l2_book: self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, bids=bids, asks=asks) else: self._l2_book[pair].book.bids = bids self._l2_book[pair].book.asks = asks await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg) async def _pair_level2_update(self, msg: dict, timestamp: float): pair = self.exchange_symbol_to_std_symbol(msg['product_id']) ts = self.timestamp_normalize(msg['time']) delta = {BID: [], ASK: []} for side, price, amount in msg['changes']: side = BID if side == 'buy' else ASK price = Decimal(price) amount = Decimal(amount) if amount == 0: del self._l2_book[pair].book[side][price] delta[side].append((price, 0)) else: self._l2_book[pair].book[side][price] = amount delta[side].append((price, amount)) await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=ts, raw=msg, delta=delta) async def _book_snapshot(self, pairs: list): # Coinbase needs some time to send messages to us # before we request the snapshot. If we don't sleep # the snapshot seq no could be much earlier than # the subsequent messages, causing a seq no mismatch. await asyncio.sleep(2) urls = [self.rest_endpoints[0].route('l3book', self.sandbox).format(pair) for pair in pairs] results = [] for url in urls: ret = await self.http_conn.read(url) results.append(ret) # rate limit - 3 per second await asyncio.sleep(0.3) timestamp = time.time() for res, pair in zip(results, pairs): orders = json.loads(res, parse_float=Decimal) npair = self.exchange_symbol_to_std_symbol(pair) self._l3_book[npair] = OrderBook(self.id, pair, max_depth=self.max_depth) self.seq_no[npair] = orders['sequence'] for side in (BID, ASK): for price, size, order_id in orders[side + 's']: price = Decimal(price) size = Decimal(size) if price in self._l3_book[npair].book[side]: self._l3_book[npair].book[side][price][order_id] = size else: self._l3_book[npair].book[side][price] = {order_id: size} self.order_map[order_id] = (price, size) await self.book_callback(L3_BOOK, self._l3_book[npair], timestamp, raw=orders) async def _open(self, msg: dict, timestamp: float): if not self.keep_l3_book: return delta = {BID: [], ASK: []} price = Decimal(msg['price']) side = ASK if msg['side'] == 'sell' else BID size = Decimal(msg['remaining_size']) pair = self.exchange_symbol_to_std_symbol(msg['product_id']) order_id = msg['order_id'] ts = self.timestamp_normalize(msg['time']) if price in self._l3_book[pair].book[side]: self._l3_book[pair].book[side][price][order_id] = size else: self._l3_book[pair].book[side][price] = {order_id: size} self.order_map[order_id] = (price, size) delta[side].append((order_id, price, size)) await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=ts, delta=delta, raw=msg, sequence_number=msg['sequence']) async def _done(self, msg: dict, timestamp: float): """ per Coinbase API Docs: A done message will be sent for received orders which are fully filled or canceled due to self-trade prevention. There will be no open message for such orders. Done messages for orders which are not on the book should be ignored when maintaining a real-time order book. """ if 'price' not in msg: # market order life cycle: received -> done self.order_type_map.pop(msg['order_id'], None) self.order_map.pop(msg['order_id'], None) return order_id = msg['order_id'] self.order_type_map.pop(order_id, None) if order_id not in self.order_map: return del self.order_map[order_id] if self.keep_l3_book: delta = {BID: [], ASK: []} price = Decimal(msg['price']) side = ASK if msg['side'] == 'sell' else BID pair = self.exchange_symbol_to_std_symbol(msg['product_id']) ts = self.timestamp_normalize(msg['time']) del self._l3_book[pair].book[side][price][order_id] if len(self._l3_book[pair].book[side][price]) == 0: del self._l3_book[pair].book[side][price] delta[side].append((order_id, price, 0)) await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, delta=delta, timestamp=ts, raw=msg, sequence_number=msg['sequence']) async def _received(self, msg: dict, timestamp: float): """ per Coinbase docs: A valid order has been received and is now active. This message is emitted for every single valid order as soon as the matching engine receives it whether it fills immediately or not. This message is the only time we receive the order type (limit vs market) for a given order, so we keep it in a map by order ID. """ order_id = msg["order_id"] order_type = msg["order_type"] self.order_type_map[order_id] = order_type async def _change(self, msg: dict, timestamp: float): """ Like done, these updates can be sent for orders that are not in the book. Per the docs: Not all done or change messages will result in changing the order book. These messages will be sent for received orders which are not yet on the order book. Do not alter the order book for such messages, otherwise your order book will be incorrect. """ if not self.keep_l3_book: return delta = {BID: [], ASK: []} if 'price' not in msg or not msg['price']: return order_id = msg['order_id'] if order_id not in self.order_map: return ts = self.timestamp_normalize(msg['time']) price = Decimal(msg['price']) side = ASK if msg['side'] == 'sell' else BID new_size = Decimal(msg['new_size']) pair = self.exchange_symbol_to_std_symbol(msg['product_id']) self._l3_book[pair].book[side][price][order_id] = new_size self.order_map[order_id] = (price, new_size) delta[side].append((order_id, price, new_size)) await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, delta=delta, timestamp=ts, raw=msg, sequence_number=msg['sequence']) async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float): # PERF perf_start(self.id, 'msg') msg = json.loads(msg, parse_float=Decimal) if self.seq_no: if 'product_id' in msg and 'sequence' in msg: pair = self.exchange_symbol_to_std_symbol(msg['product_id']) if not self.seq_no.get(pair, None): return if msg['sequence'] <= self.seq_no[pair]: return if msg['sequence'] != self.seq_no[pair] + 1: LOG.warning("%s: Missing sequence number detected for %s. Received %d, expected %d", self.id, pair, msg['sequence'], self.seq_no[pair] + 1) LOG.warning("%s: Resetting data for %s", self.id, pair) self.__reset(symbol=pair) await self._book_snapshot([pair]) return self.seq_no[pair] = msg['sequence'] if 'type' in msg: if msg['type'] == 'ticker': await self._ticker(msg, timestamp) elif msg['type'] == 'match' or msg['type'] == 'last_match': await self._book_update(msg, timestamp) elif msg['type'] == 'snapshot': await self._pair_level2_snapshot(msg, timestamp) elif msg['type'] == 'l2update': await self._pair_level2_update(msg, timestamp) elif msg['type'] == 'open': await self._open(msg, timestamp) elif msg['type'] == 'done': await self._done(msg, timestamp) elif msg['type'] == 'change': await self._change(msg, timestamp) elif msg['type'] == 'received': await self._received(msg, timestamp) elif msg['type'] == 'activate': pass elif msg['type'] == 'subscriptions': pass else: LOG.warning("%s: Invalid message type %s", self.id, msg) # PERF perf_end(self.id, 'msg') # PERF perf_log(self.id, 'msg') async def subscribe(self, conn: AsyncConnection): self.__reset() for chan in self.subscription: await conn.write(json.dumps({"type": "subscribe", "product_ids": self.subscription[chan], "channels": [chan] })) chan = self.std_channel_to_exchange(L3_BOOK) if chan in self.subscription: await self._book_snapshot(self.subscription[chan])
class HuobiDM(Feed): id = HUOBI_DM websocket_endpoints = [WebsocketEndpoint('wss://www.hbdm.com/ws')] rest_endpoints = [ RestEndpoint('https://www.hbdm.com', routes=Routes('/api/v1/contract_contract_info')) ] websocket_channels = { L2_BOOK: 'depth.step0', TRADES: 'trade.detail', } @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for e in data['data']: # Pricing is all in USD, see https://huobiglobal.zendesk.com/hc/en-us/articles/360000113102-Introduction-of-Huobi-Futures s = Symbol(e['symbol'], 'USD', type=FUTURES, expiry_date=e['contract_code'].replace(e['symbol'], '')) ret[s.normalized] = e['contract_code'] info['tick_size'][s.normalized] = e['price_tick'] info['instrument_type'][s.normalized] = FUTURES return ret, info def __reset(self): self._l2_book = {} async def _book(self, msg: dict, timestamp: float): """ { 'ch':'market.BTC_CW.depth.step0', 'ts':1565857755564, 'tick':{ 'mrid':14848858327, 'id':1565857755, 'bids':[ [ Decimal('9829.99'), 1], ... ] 'asks':[ [ 9830, 625], ... ] }, 'ts':1565857755552, 'version':1565857755, 'ch':'market.BTC_CW.depth.step0' } """ pair = self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1]) data = msg['tick'] # When Huobi Delists pairs, empty updates still sent: # {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951241196, 'tick': {'mrid': 50651100044, 'id': 1606951241, 'ts': 1606951241195, 'version': 1606951241, 'ch': 'market.AKRO-USD.depth.step0'}} # {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951242297, 'tick': {'mrid': 50651100044, 'id': 1606951242, 'ts': 1606951242295, 'version': 1606951242, 'ch': 'market.AKRO-USD.depth.step0'}} if 'bids' in data and 'asks' in data: if pair not in self._l2_book: self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) self._l2_book[pair].book.bids = { Decimal(price): Decimal(amount) for price, amount in data['bids'] } self._l2_book[pair].book.asks = { Decimal(price): Decimal(amount) for price, amount in data['asks'] } await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize( msg['ts']), raw=msg) async def _trade(self, msg: dict, timestamp: float): """ { 'ch': 'market.btcusd.trade.detail', 'ts': 1549773923965, 'tick': { 'id': 100065340982, 'ts': 1549757127140, 'data': [{'id': '10006534098224147003732', 'amount': Decimal('0.0777'), 'price': Decimal('3669.69'), 'direction': 'buy', 'ts': 1549757127140}]} } """ for trade in msg['tick']['data']: t = Trade(self.id, self.exchange_symbol_to_std_symbol( msg['ch'].split('.')[1]), BUY if trade['direction'] == 'buy' else SELL, Decimal(trade['amount']), Decimal(trade['price']), self.timestamp_normalize(trade['ts']), id=str(trade['id']), raw=trade) await self.callback(TRADES, t, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): # unzip message msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS) msg = json.loads(msg, parse_float=Decimal) # Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it if 'ping' in msg: await conn.write(json.dumps({'pong': msg['ping']})) elif 'status' in msg and msg['status'] == 'ok': return elif 'ch' in msg: if 'trade' in msg['ch']: await self._trade(msg, timestamp) elif 'depth' in msg['ch']: await self._book(msg, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() client_id = 0 for chan, symbols in conn.subscription.items(): for symbol in symbols: client_id += 1 await conn.write( json.dumps({ "sub": f"market.{symbol}.{chan}", "id": str(client_id) }))
class Bitmex(Feed, BitmexRestMixin): id = BITMEX websocket_endpoints = [WebsocketEndpoint('wss://www.bitmex.com/realtime', sandbox='wss://testnet.bitmex.com/realtime', options={'compression': None})] rest_endpoints = [RestEndpoint('https://www.bitmex.com', routes=Routes('/api/v1/instrument/active'))] websocket_channels = { L2_BOOK: 'orderBookL2', TRADES: 'trade', TICKER: 'quote', FUNDING: 'funding', ORDER_INFO: 'order', OPEN_INTEREST: 'instrument', LIQUIDATIONS: 'liquidation' } request_limit = 0.5 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for entry in data: base = entry['rootSymbol'].replace("XBT", "BTC") quote = entry['quoteCurrency'].replace("XBT", "BTC") stype = PERPETUAL if entry['expiry']: stype = FUTURES s = Symbol(base, quote, type=stype, expiry_date=entry['expiry']) ret[s.normalized] = entry['symbol'] info['tick_size'][s.normalized] = entry['tickSize'] info['instrument_type'][s.normalized] = stype return ret, info def _reset(self): self.partial_received = defaultdict(bool) self.order_id = {} self.open_orders = {} for pair in self.normalized_symbols: self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) self.order_id[pair] = defaultdict(dict) @staticmethod def normalize_order_status(status): status_map = { 'New': OPEN, 'Filled': FILLED, 'Canceled': CANCELLED, } return status_map[status] def init_order_info(self, o): oi = OrderInfo( self.id, self.exchange_symbol_to_std_symbol(o['symbol']), o['orderID'], BUY if o['side'] == 'Buy' else SELL, self.normalize_order_status(o['ordStatus']), LIMIT if o['ordType'].lower() == 'limit' else MARKET if o['ordType'].lower() == 'market' else None, Decimal(o['avgPx']) if o['avgPx'] else Decimal(o['price']), Decimal(o['orderQty']), Decimal(o['leavesQty']), self.timestamp_normalize(o['timestamp']), raw=str(o), # Need to convert to string to avoid json serialization error when updating order ) return oi async def _order(self, msg: dict, timestamp: float): """ order msg example { "table": "order", "action": "partial", "keys": [ "orderID" ], "types": { "orderID": "guid", "clOrdID": "string", "clOrdLinkID": "symbol", "account": "long", "symbol": "symbol", "side": "symbol", "simpleOrderQty": "float", "orderQty": "long", "price": "float", "displayQty": "long", "stopPx": "float", "pegOffsetValue": "float", "pegPriceType": "symbol", "currency": "symbol", "settlCurrency": "symbol", "ordType": "symbol", "timeInForce": "symbol", "execInst": "symbol", "contingencyType": "symbol", "exDestination": "symbol", "ordStatus": "symbol", "triggered": "symbol", "workingIndicator": "boolean", "ordRejReason": "symbol", "simpleLeavesQty": "float", "leavesQty": "long", "simpleCumQty": "float", "cumQty": "long", "avgPx": "float", "multiLegReportingType": "symbol", "text": "string", "transactTime": "timestamp", "timestamp": "timestamp" }, "foreignKeys": { "symbol": "instrument", "side": "side", "ordStatus": "ordStatus" }, "attributes": { "orderID": "grouped", "account": "grouped", "ordStatus": "grouped", "workingIndicator": "grouped" }, "filter": { "account": 1600000, "symbol": "ETHUSDTH22" }, "data": [ { "orderID": "360fad5a-49e3-4187-ad04-8fac82b8a95f", "clOrdID": "", "clOrdLinkID": "", "account": 1600000, "symbol": "ETHUSDTH22", "side": "Buy", "simpleOrderQty": null, "orderQty": 1000, "price": 2000, "displayQty": null, "stopPx": null, "pegOffsetValue": null, "pegPriceType": "", "currency": "USDT", "settlCurrency": "USDt", "ordType": "Limit", "timeInForce": "GoodTillCancel", "execInst": "", "contingencyType": "", "exDestination": "XBME", "ordStatus": "New", "triggered": "", "workingIndicator": true, "ordRejReason": "", "simpleLeavesQty": null, "leavesQty": 1000, "simpleCumQty": null, "cumQty": 0, "avgPx": null, "multiLegReportingType": "SingleSecurity", "text": "Submitted via API.", "transactTime": "2022-02-13T00:15:02.570000Z", "timestamp": "2022-02-13T00:15:02.570000Z" }, { "orderID": "74d2ad0a-49f1-44dc-820f-5f0cfd64c1a3", "clOrdID": "", "clOrdLinkID": "", "account": 1600000, "symbol": "ETHUSDTH22", "side": "Buy", "simpleOrderQty": null, "orderQty": 1000, "price": 2000, "displayQty": null, "stopPx": null, "pegOffsetValue": null, "pegPriceType": "", "currency": "USDT", "settlCurrency": "USDt", "ordType": "Limit", "timeInForce": "GoodTillCancel", "execInst": "", "contingencyType": "", "exDestination": "XBME", "ordStatus": "New", "triggered": "", "workingIndicator": true, "ordRejReason": "", "simpleLeavesQty": null, "leavesQty": 1000, "simpleCumQty": null, "cumQty": 0, "avgPx": null, "multiLegReportingType": "SingleSecurity", "text": "Submitted via API.", "transactTime": "2022-02-13T00:17:13.796000Z", "timestamp": "2022-02-13T00:17:13.796000Z" } ] } { "table": "order", "action": "insert", "data": [ { "orderID": "0c4e4a8e-b234-495f-8b94-c4766786c4a5", "clOrdID": "", "clOrdLinkID": "", "account": 1600000, "symbol": "ETHUSDTH22", "side": "Buy", "simpleOrderQty": null, "orderQty": 1000, "price": 2000, "displayQty": null, "stopPx": null, "pegOffsetValue": null, "pegPriceType": "", "currency": "USDT", "settlCurrency": "USDt", "ordType": "Limit", "timeInForce": "GoodTillCancel", "execInst": "", "contingencyType": "", "exDestination": "XBME", "ordStatus": "New", "triggered": "", "workingIndicator": true, "ordRejReason": "", "simpleLeavesQty": null, "leavesQty": 1000, "simpleCumQty": null, "cumQty": 0, "avgPx": null, "multiLegReportingType": "SingleSecurity", "text": "Submitted via API.", "transactTime": "2022-02-13T00:21:50.268000Z", "timestamp": "2022-02-13T00:21:50.268000Z" } ] } { "table": "order", "action": "update", "data": [ { "orderID": "360fa95a-49e3-4187-ad04-8fac82b8a95f", "ordStatus": "Canceled", "workingIndicator": false, "leavesQty": 0, "text": "Canceled: Cancel from www.bitmex.com\nSubmitted via API.", "timestamp": "2022-02-13T08:16:36.446000Z", "clOrdID": "", "account": 1600000, "symbol": "ETHUSDTH22" } ] } """ if msg['action'] == 'partial': # Initial snapshot of open orders self.open_orders = {} for o in msg['data']: oi = self.init_order_info(o) self.open_orders[oi.id] = oi elif msg['action'] == 'insert': for o in msg['data']: oi = self.init_order_info(o) self.open_orders[oi.id] = oi await self.callback(ORDER_INFO, oi, timestamp) elif msg['action'] == 'update': for o in msg['data']: oi = self.open_orders.get(o['orderID']) if oi: info = oi.to_dict() if 'ordStatus' in o: info['status'] = self.normalize_order_status(o['ordStatus']) if 'leaveQty' in o: info['remaining'] = Decimal(o['leavesQty']) if 'avgPx' in o: info['price'] = Decimal(o['avgPx']) info['raw'] = str(o) # Not sure if this is needed new_oi = OrderInfo(**info) if new_oi.status in (FILLED, CANCELLED): self.open_orders.pop(new_oi.id) else: self.open_orders[new_oi.id] = oi await self.callback(ORDER_INFO, new_oi, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg) async def _trade(self, msg: dict, timestamp: float): """ trade msg example { 'timestamp': '2018-05-19T12:25:26.632Z', 'symbol': 'XBTUSD', 'side': 'Buy', 'size': 40, 'price': 8335, 'tickDirection': 'PlusTick', 'trdMatchID': '5f4ecd49-f87f-41c0-06e3-4a9405b9cdde', 'grossValue': 479920, 'homeNotional': Decimal('0.0047992'), 'foreignNotional': 40 } """ for data in msg['data']: ts = self.timestamp_normalize(data['timestamp']) t = Trade( self.id, self.exchange_symbol_to_std_symbol(data['symbol']), BUY if data['side'] == 'Buy' else SELL, Decimal(data['size']), Decimal(data['price']), ts, id=data['trdMatchID'], raw=data ) await self.callback(TRADES, t, timestamp) async def _book(self, msg: dict, timestamp: float): """ the Full bitmex book """ # PERF perf_start(self.id, 'book_msg') if not msg['data']: # see https://github.com/bmoscon/cryptofeed/issues/688 # msg['data'] can be an empty list return delta = None # if we reset the book, force a full update pair = self.exchange_symbol_to_std_symbol(msg['data'][0]['symbol']) if not self.partial_received[pair]: # per bitmex documentation messages received before partial # should be discarded if msg['action'] != 'partial': return self.partial_received[pair] = True if msg['action'] == 'partial': for data in msg['data']: side = BID if data['side'] == 'Buy' else ASK price = Decimal(data['price']) size = Decimal(data['size']) order_id = data['id'] self._l2_book[pair].book[side][price] = size self.order_id[pair][side][order_id] = price elif msg['action'] == 'insert': delta = {BID: [], ASK: []} for data in msg['data']: side = BID if data['side'] == 'Buy' else ASK price = Decimal(data['price']) size = Decimal(data['size']) order_id = data['id'] self._l2_book[pair].book[side][price] = size self.order_id[pair][side][order_id] = price delta[side].append((price, size)) elif msg['action'] == 'update': delta = {BID: [], ASK: []} for data in msg['data']: side = BID if data['side'] == 'Buy' else ASK update_size = Decimal(data['size']) order_id = data['id'] price = self.order_id[pair][side][order_id] self._l2_book[pair].book[side][price] = update_size self.order_id[pair][side][order_id] = price delta[side].append((price, update_size)) elif msg['action'] == 'delete': delta = {BID: [], ASK: []} for data in msg['data']: side = BID if data['side'] == 'Buy' else ASK order_id = data['id'] delete_price = self.order_id[pair][side][order_id] del self.order_id[pair][side][order_id] del self._l2_book[pair].book[side][delete_price] delta[side].append((delete_price, 0)) else: LOG.warning("%s: Unexpected l2 Book message %s", self.id, msg) return # PERF perf_end(self.id, 'book_msg') # PERF perf_log(self.id, 'book_msg') await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg, delta=delta) async def _ticker(self, msg: dict, timestamp: float): for data in msg['data']: t = Ticker( self.id, self.exchange_symbol_to_std_symbol(data['symbol']), Decimal(data['bidPrice']), Decimal(data['askPrice']), self.timestamp_normalize(data['timestamp']), raw=data ) await self.callback(TICKER, t, timestamp) async def _funding(self, msg: dict, timestamp: float): """ {'table': 'funding', 'action': 'partial', 'keys': ['timestamp', 'symbol'], 'types': { 'timestamp': 'timestamp', 'symbol': 'symbol', 'fundingInterval': 'timespan', 'fundingRate': 'float', 'fundingRateDaily': 'float' }, 'foreignKeys': { 'symbol': 'instrument' }, 'attributes': { 'timestamp': 'sorted', 'symbol': 'grouped' }, 'filter': {'symbol': 'XBTUSD'}, 'data': [{ 'timestamp': '2018-08-21T20:00:00.000Z', 'symbol': 'XBTUSD', 'fundingInterval': '2000-01-01T08:00:00.000Z', 'fundingRate': Decimal('-0.000561'), 'fundingRateDaily': Decimal('-0.001683') }] } """ for data in msg['data']: ts = self.timestamp_normalize(data['timestamp']) interval = data['fundingInterval'] f = Funding( self.id, self.exchange_symbol_to_std_symbol(data['symbol']), None, data['fundingRate'], self.timestamp_normalize(data['timestamp'] + timedelta(hours=interval.hour)), ts, raw=data ) await self.callback(FUNDING, f, timestamp) async def _instrument(self, msg: dict, timestamp: float): """ Example instrument data { 'table':'instrument', 'action':'partial', 'keys':[ 'symbol' ], 'types':{ 'symbol':'symbol', 'rootSymbol':'symbol', 'state':'symbol', 'typ':'symbol', 'listing':'timestamp', 'front':'timestamp', 'expiry':'timestamp', 'settle':'timestamp', 'relistInterval':'timespan', 'inverseLeg':'symbol', 'sellLeg':'symbol', 'buyLeg':'symbol', 'optionStrikePcnt':'float', 'optionStrikeRound':'float', 'optionStrikePrice':'float', 'optionMultiplier':'float', 'positionCurrency':'symbol', 'underlying':'symbol', 'quoteCurrency':'symbol', 'underlyingSymbol':'symbol', 'reference':'symbol', 'referenceSymbol':'symbol', 'calcInterval':'timespan', 'publishInterval':'timespan', 'publishTime':'timespan', 'maxOrderQty':'long', 'maxPrice':'float', 'lotSize':'long', 'tickSize':'float', 'multiplier':'long', 'settlCurrency':'symbol', 'underlyingToPositionMultiplier':'long', 'underlyingToSettleMultiplier':'long', 'quoteToSettleMultiplier':'long', 'isQuanto':'boolean', 'isInverse':'boolean', 'initMargin':'float', 'maintMargin':'float', 'riskLimit':'long', 'riskStep':'long', 'limit':'float', 'capped':'boolean', 'taxed':'boolean', 'deleverage':'boolean', 'makerFee':'float', 'takerFee':'float', 'settlementFee':'float', 'insuranceFee':'float', 'fundingBaseSymbol':'symbol', 'fundingQuoteSymbol':'symbol', 'fundingPremiumSymbol':'symbol', 'fundingTimestamp':'timestamp', 'fundingInterval':'timespan', 'fundingRate':'float', 'indicativeFundingRate':'float', 'rebalanceTimestamp':'timestamp', 'rebalanceInterval':'timespan', 'openingTimestamp':'timestamp', 'closingTimestamp':'timestamp', 'sessionInterval':'timespan', 'prevClosePrice':'float', 'limitDownPrice':'float', 'limitUpPrice':'float', 'bankruptLimitDownPrice':'float', 'bankruptLimitUpPrice':'float', 'prevTotalVolume':'long', 'totalVolume':'long', 'volume':'long', 'volume24h':'long', 'prevTotalTurnover':'long', 'totalTurnover':'long', 'turnover':'long', 'turnover24h':'long', 'homeNotional24h':'float', 'foreignNotional24h':'float', 'prevPrice24h':'float', 'vwap':'float', 'highPrice':'float', 'lowPrice':'float', 'lastPrice':'float', 'lastPriceProtected':'float', 'lastTickDirection':'symbol', 'lastChangePcnt':'float', 'bidPrice':'float', 'midPrice':'float', 'askPrice':'float', 'impactBidPrice':'float', 'impactMidPrice':'float', 'impactAskPrice':'float', 'hasLiquidity':'boolean', 'openInterest':'long', 'openValue':'long', 'fairMethod':'symbol', 'fairBasisRate':'float', 'fairBasis':'float', 'fairPrice':'float', 'markMethod':'symbol', 'markPrice':'float', 'indicativeTaxRate':'float', 'indicativeSettlePrice':'float', 'optionUnderlyingPrice':'float', 'settledPrice':'float', 'timestamp':'timestamp' }, 'foreignKeys':{ 'inverseLeg':'instrument', 'sellLeg':'instrument', 'buyLeg':'instrument' }, 'attributes':{ 'symbol':'unique' }, 'filter':{ 'symbol':'XBTUSD' }, 'data':[ { 'symbol':'XBTUSD', 'rootSymbol':'XBT', 'state':'Open', 'typ':'FFWCSX', 'listing':'2016-05-13T12:00:00.000Z', 'front':'2016-05-13T12:00:00.000Z', 'expiry':None, 'settle':None, 'relistInterval':None, 'inverseLeg':'', 'sellLeg':'', 'buyLeg':'', 'optionStrikePcnt':None, 'optionStrikeRound':None, 'optionStrikePrice':None, 'optionMultiplier':None, 'positionCurrency':'USD', 'underlying':'XBT', 'quoteCurrency':'USD', 'underlyingSymbol':'XBT=', 'reference':'BMEX', 'referenceSymbol':'.BXBT', 'calcInterval':None, 'publishInterval':None, 'publishTime':None, 'maxOrderQty':10000000, 'maxPrice':1000000, 'lotSize':1, 'tickSize':Decimal( '0.5' ), 'multiplier':-100000000, 'settlCurrency':'XBt', 'underlyingToPositionMultiplier':None, 'underlyingToSettleMultiplier':-100000000, 'quoteToSettleMultiplier':None, 'isQuanto':False, 'isInverse':True, 'initMargin':Decimal( '0.01' ), 'maintMargin':Decimal( '0.005' ), 'riskLimit':20000000000, 'riskStep':10000000000, 'limit':None, 'capped':False, 'taxed':True, 'deleverage':True, 'makerFee':Decimal( '-0.00025' ), 'takerFee':Decimal( '0.00075' ), 'settlementFee':0, 'insuranceFee':0, 'fundingBaseSymbol':'.XBTBON8H', 'fundingQuoteSymbol':'.USDBON8H', 'fundingPremiumSymbol':'.XBTUSDPI8H', 'fundingTimestamp':'2020-02-02T04:00:00.000Z', 'fundingInterval':'2000-01-01T08:00:00.000Z', 'fundingRate':Decimal( '0.000106' ), 'indicativeFundingRate':Decimal( '0.0001' ), 'rebalanceTimestamp':None, 'rebalanceInterval':None, 'openingTimestamp':'2020-02-02T00:00:00.000Z', 'closingTimestamp':'2020-02-02T01:00:00.000Z', 'sessionInterval':'2000-01-01T01:00:00.000Z', 'prevClosePrice':Decimal( '9340.63' ), 'limitDownPrice':None, 'limitUpPrice':None, 'bankruptLimitDownPrice':None, 'bankruptLimitUpPrice':None, 'prevTotalVolume':1999389257669, 'totalVolume':1999420432348, 'volume':31174679, 'volume24h':1605909209, 'prevTotalTurnover':27967114248663460, 'totalTurnover':27967447182062520, 'turnover':332933399058, 'turnover24h':17126993087717, 'homeNotional24h':Decimal( '171269.9308771703' ), 'foreignNotional24h':1605909209, 'prevPrice24h':9348, 'vwap':Decimal( '9377.3443' ), 'highPrice':9464, 'lowPrice':Decimal( '9287.5' ), 'lastPrice':9352, 'lastPriceProtected':9352, 'lastTickDirection':'ZeroMinusTick', 'lastChangePcnt':Decimal( '0.0004' ), 'bidPrice':9352, 'midPrice':Decimal( '9352.25' ), 'askPrice':Decimal( '9352.5' ), 'impactBidPrice':Decimal( '9351.9125' ), 'impactMidPrice':Decimal( '9352.25' ), 'impactAskPrice':Decimal( '9352.7871' ), 'hasLiquidity':True, 'openInterest':983043322, 'openValue':10518563545400, 'fairMethod':'FundingRate', 'fairBasisRate':Decimal( '0.11607' ), 'fairBasis':Decimal( '0.43' ), 'fairPrice':Decimal( '9345.36' ), 'markMethod':'FairPrice', 'markPrice':Decimal( '9345.36' ), 'indicativeTaxRate':0, 'indicativeSettlePrice':Decimal( '9344.93' ), 'optionUnderlyingPrice':None, 'settledPrice':None, 'timestamp':'2020-02-02T00:30:43.772Z' } ] } """ for data in msg['data']: if 'openInterest' in data: ts = self.timestamp_normalize(data['timestamp']) oi = OpenInterest(self.id, self.exchange_symbol_to_std_symbol(data['symbol']), Decimal(data['openInterest']), ts, raw=data) await self.callback(OPEN_INTEREST, oi, timestamp) async def _liquidation(self, msg: dict, timestamp: float): """ liquidation msg example { 'orderID': '9513c849-ca0d-4e11-8190-9d221972288c', 'symbol': 'XBTUSD', 'side': 'Buy', 'price': 6833.5, 'leavesQty': 2020 } """ if msg['action'] == 'insert': for data in msg['data']: liq = Liquidation( self.id, self.exchange_symbol_to_std_symbol(data['symbol']), BUY if data['side'] == 'Buy' else SELL, Decimal(data['leavesQty']), Decimal(data['price']), data['orderID'], UNFILLED, None, raw=data ) await self.callback(LIQUIDATIONS, liq, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'table' in msg: if msg['table'] == 'trade': await self._trade(msg, timestamp) elif msg['table'] == 'order': await self._order(msg, timestamp) elif msg['table'] == 'orderBookL2': await self._book(msg, timestamp) elif msg['table'] == 'funding': await self._funding(msg, timestamp) elif msg['table'] == 'instrument': await self._instrument(msg, timestamp) elif msg['table'] == 'quote': await self._ticker(msg, timestamp) elif msg['table'] == 'liquidation': await self._liquidation(msg, timestamp) else: LOG.warning("%s: Unhandled table=%r in %r", conn.uuid, msg['table'], msg) elif 'info' in msg: LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg) elif 'subscribe' in msg: if not msg['success']: LOG.error("%s: Subscribe failure: %s", conn.uuid, msg) elif 'error' in msg: LOG.error("%s: Error message from exchange: %s", conn.uuid, msg) elif 'request' in msg: if msg['success']: LOG.debug("%s: Success %s", conn.uuid, msg['request'].get('op')) else: LOG.warning("%s: Failure %s", conn.uuid, msg['request']) else: LOG.warning("%s: Unexpected message from exchange: %s", conn.uuid, msg) async def subscribe(self, conn: AsyncConnection): self._reset() await self._authenticate(conn) chans = [] for chan in self.subscription: for pair in self.subscription[chan]: chans.append(f"{chan}:{pair}") for i in range(0, len(chans), 10): await conn.write(json.dumps({"op": "subscribe", "args": chans[i:i + 10]})) async def _authenticate(self, conn: AsyncConnection): """Send API Key with signed message.""" # Docs: https://www.bitmex.com/app/apiKeys # https://github.com/BitMEX/sample-market-maker/blob/master/test/websocket-apikey-auth-test.py if self.key_id and self.key_secret: LOG.info('%s: Authenticate with signature', conn.uuid) expires = int(time.time()) + 365 * 24 * 3600 # One year msg = f'GET/realtime{expires}'.encode('utf-8') signature = hmac.new(self.key_secret.encode('utf-8'), msg, digestmod=hashlib.sha256).hexdigest() await conn.write(json.dumps({'op': 'authKeyExpires', 'args': [self.key_id, expires, signature]}))
class Probit(Feed): id = PROBIT websocket_endpoints = [ WebsocketEndpoint('wss://api.probit.com/api/exchange/v1/ws') ] rest_endpoints = [ RestEndpoint('https://api.probit.com', routes=Routes('/api/exchange/v1/market')) ] websocket_channels = { L2_BOOK: 'order_books', TRADES: 'recent_trades', } @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = {'instrument_type': {}} # doc: https://docs-en.probit.com/reference-link/market for entry in data['data']: if entry['closed']: continue s = Symbol(entry['base_currency_id'], entry['quote_currency_id']) ret[s.normalized] = entry['id'] info['instrument_type'][s.normalized] = s.type return ret, info def __reset(self): self._l2_book = {} async def _trades(self, msg: dict, timestamp: float): ''' { "channel":"marketdata", "market_id":"ETH-BTC", "status":"ok","lag":0, "recent_trades":[ { "id":"ETH-BTC:4429182", "price":"0.028229", "quantity":"3.117", "time":"2020-11-01T03:59:06.277Z", "side":"buy","tick_direction":"down" },{ "id":"ETH-BTC:4429183", "price":"0.028227", "quantity":"1.793", "time":"2020-11-01T03:59:14.528Z", "side":"buy", "tick_direction":"down" } ],"reset":true } { "channel":"marketdata", "market_id":"ETH-BTC", "status":"ok","lag":0, "recent_trades":[ { "id":"ETH-BTC:4429282", "price":"0.028235", "quantity":"2.203", "time":"2020-11-01T04:22:15.117Z", "side":"buy", "tick_direction":"down" } ] } ''' pair = self.exchange_symbol_to_std_symbol(msg['market_id']) for update in msg['recent_trades']: t = Trade(self.id, pair, BUY if update['side'] == 'buy' else SELL, Decimal(update['quantity']), Decimal(update['price']), self.timestamp_normalize(update['time']), id=update['id'], raw=update) await self.callback(TRADES, t, timestamp) async def _l2_update(self, msg: dict, timestamp: float): ''' { "channel":"marketdata", "market_id":"ETH-BTC", "status":"ok", "lag":0, "order_books":[ { "side":"buy", "price":"0.0165", "quantity":"0.47" },{ "side":"buy", "price":"0", "quantity":"14656.177" },{ "side":"sell", "price":"6400", "quantity":"0.001" }], "reset":true } { "channel":"marketdata", "market_id":"ETH-BTC", "status":"ok", "lag":0, "order_books":[ { "side":"buy", "price":"0.0281", "quantity":"48.541" },{ "side":"sell", "price":"0.0283", "quantity":"0" }] } ''' pair = self.exchange_symbol_to_std_symbol(msg['market_id']) is_snapshot = msg.get('reset', False) if is_snapshot: self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) for entry in msg["order_books"]: price = Decimal(entry['price']) quantity = Decimal(entry['quantity']) side = BID if entry['side'] == "buy" else ASK self._l2_book[pair].book[side][price] = quantity await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg) else: delta = {BID: [], ASK: []} for entry in msg["order_books"]: price = Decimal(entry['price']) quantity = Decimal(entry['quantity']) side = BID if entry['side'] == "buy" else ASK if quantity == 0: if price in self._l2_book[pair].book[side]: del self._l2_book[pair].book[side][price] delta[side].append((price, 0)) else: self._l2_book[pair].book[side][price] = quantity delta[side].append((price, quantity)) await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg, delta=delta) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) # Probit can send multiple type updates in one message so we avoid the use of elif if 'recent_trades' in msg: await self._trades(msg, timestamp) if 'order_books' in msg: await self._l2_update(msg, timestamp) # Probit has a 'ticker' channel, but it provide OHLC-last data, not BBO px. async def subscribe(self, conn: AsyncConnection): self.__reset() if self.subscription: for chan in self.subscription: for pair in self.subscription[chan]: await conn.write( json.dumps({ "type": "subscribe", "channel": "marketdata", "filter": [chan], "interval": 100, "market_id": pair, }))
class EXX(Feed): id = EXX_id websocket_endpoints = [WebsocketEndpoint('wss://ws.exx.com/websocket')] rest_endpoints = [ RestEndpoint('https://api.exx.com', routes=Routes('/data/v1/tickers')) ] websocket_channels = { L2_BOOK: 'ENTRUST_ADD', TRADES: 'TRADE', } @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = {'instrument_type': {}} exchange = [key.upper() for key in data.keys()] for sym in exchange: b, q = sym.split("_") s = Symbol(b, q) ret[s.normalized] = sym info['instrument_type'][s.normalized] = s.type return ret, info def __reset(self): self._l2_book = {} async def _book_update(self, msg: dict, timestamp: float): """ Snapshot: [ [ 'AE', '1', 'BTC_USDT', '1547941504', { 'asks':[ [ '25000.00000000', '0.02000000' ], [ '19745.83000000', '0.00200000' ], [ '19698.96000000', '0.00100000' ], ... ] }, { 'bids':[ [ '3662.83040000', '0.00100000' ], [ '3662.77540000', '0.01000000' ], [ '3662.59900000', '0.10300000' ], ... ] } ] ] Update: ['E', '1', '1547942636', 'BTC_USDT', 'ASK', '3674.91740000', '0.02600000'] """ delta = {BID: [], ASK: []} if msg[0] == 'AE': # snapshot delta = None pair = self.exchange_symbol_to_std_symbol(msg[2]) ts = msg[3] asks = msg[4]['asks'] if 'asks' in msg[4] else msg[5]['asks'] bids = msg[5]['bids'] if 'bids' in msg[5] else msg[4]['bids'] self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) self._l2_book[pair].book.bids = { Decimal(price): Decimal(amount) for price, amount in bids } self._l2_book[pair].book.asks = { Decimal(price): Decimal(amount) for price, amount in asks } else: # Update ts = msg[2] pair = self.exchange_symbol_to_std_symbol(msg[3]) side = ASK if msg[4] == 'ASK' else BID price = Decimal(msg[5]) amount = Decimal(msg[6]) if amount == 0: if price in self._l2_book[pair].book[side]: del self._l2_book[pair][side].book[price] delta[side].append((price, 0)) else: self._l2_book[pair].book[side][price] = amount delta[side].append((price, amount)) await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=ts, raw=msg, delta=delta) async def _trade(self, msg: dict, timestamp: float): """ Trade message ['T', '1', '1547947390', 'BTC_USDT', 'bid', '3683.74440000', '0.082', '33732290'] """ pair = self.exchange_symbol_to_std_symbol(msg[3]) t = Trade(self.id, pair, BUY if msg[4] == 'bid' else SELL, Decimal(msg[6]), Decimal(msg[5]), float(msg[2]), id=msg[7], raw=msg) await self.callback(TRADES, t, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if isinstance(msg[0], list): msg = msg[0] if msg[0] == 'E' or msg[0] == 'AE': await self._book_update(msg, timestamp) elif msg[0] == 'T': await self._trade(msg, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() for chan in self.subscription: for pair in self.subscription[chan]: await conn.write( json.dumps({ "dataType": f"1_{chan}_{pair}", "dataSize": 50, "action": "ADD" }))
class FMFW(Feed): id = FMFW_id websocket_endpoints = [ WebsocketEndpoint('wss://api.fmfw.io/api/3/ws/public') ] rest_endpoints = [ RestEndpoint('https://api.fmfw.io', routes=Routes('/api/3/public/symbol')) ] valid_candle_intervals = { '1m', '3m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '1M' } candle_interval_map = { '1m': 'M1', '3m': 'M3', '5m': 'M5', '15m': 'M15', '30m': 'M30', '1h': 'H1', '4h': 'H4', '1d': 'D1', '1w': 'D7', '1M': '1M' } websocket_channels = { L2_BOOK: 'orderbook/full', TRADES: 'trades', TICKER: 'ticker/1s', CANDLES: 'candles/' } @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for sym, symbol in data.items(): s = Symbol(symbol['base_currency'], symbol['quote_currency']) ret[s.normalized] = sym info['tick_size'][s.normalized] = symbol['tick_size'] info['instrument_type'][s.normalized] = s.type return ret, info @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 def __reset(self): self._l2_book = {} self.seq_no = {} async def _book(self, msg: dict, ts: float): if 'snapshot' in msg: for pair, update in msg['snapshot'].items(): symbol = self.exchange_symbol_to_std_symbol(pair) bids = { Decimal(price): Decimal(size) for price, size in update['b'] } asks = { Decimal(price): Decimal(size) for price, size in update['a'] } self._l2_book[symbol] = OrderBook(self.id, pair, max_depth=self.max_depth, bids=bids, asks=asks) await self.book_callback(L2_BOOK, self._l2_book[symbol], ts, sequence_number=update['s'], delta=None, raw=msg, timestamp=self.timestamp_normalize( update['t'])) self.seq_no[symbol] = update['s'] else: delta = {BID: [], ASK: []} for pair, update in msg['update'].items(): symbol = self.exchange_symbol_to_std_symbol(pair) if self.seq_no[symbol] + 1 != update['s']: raise MissingSequenceNumber self.seq_no[symbol] = update['s'] for side, key in ((BID, 'b'), (ASK, 'a')): for price, size in update[key]: price = Decimal(price) size = Decimal(size) delta[side].append((price, size)) if size == 0: del self._l2_book[symbol].book[side][price] else: self._l2_book[symbol].book[side][price] = size await self.book_callback( L2_BOOK, self._l2_book[symbol], ts, sequence_number=update['s'], delta=delta, raw=msg, timestamp=self.timestamp_normalize(update['t'])) async def _trade(self, msg: dict, ts: float): ''' { 'ch': 'trades', 'update': { 'BTCUSDT': [{ 't': 1633803835228, 'i': 1633803835228, 'p': '54774.60', 'q': '0.00004', 's': 'buy' }] } } ''' for pair, update in msg['update'].items(): symbol = self.exchange_symbol_to_std_symbol(pair) for trade in update: t = Trade(self.id, symbol, BUY if trade['s'] == 'buy' else SELL, Decimal(trade['q']), Decimal(trade['p']), self.timestamp_normalize(trade['t']), id=str(trade['i']), raw=msg) await self.callback(TRADES, t, ts) async def _ticker(self, msg: dict, ts: float): ''' { 'ch': 'ticker/1s', 'data': { 'BTCUSDT': { 't': 1633804289795, 'a': '54813.56', 'A': '0.82000', 'b': '54810.31', 'B': '0.00660', 'o': '54517.48', 'c': '54829.88', 'h': '55493.92', 'l': '53685.61', 'v': '19025.22558', 'q': '1040244549.4048389', 'p': '312.40', 'P': '0.5730272198935094', 'L': 1417964345 } } } ''' for sym, ticker in msg['data'].items(): t = Ticker(self.id, self.exchange_symbol_to_std_symbol(sym), Decimal(ticker['b']), Decimal(ticker['a']), self.timestamp_normalize(ticker['t']), raw=msg) await self.callback(TICKER, t, ts) async def _candle(self, msg: dict, ts: float): ''' { 'ch': 'candles/M1', 'update': { 'BTCUSDT': [{ 't': 1633805940000, 'o': '54849.03', 'c': '54849.03', 'h': '54849.03', 'l': '54849.03', 'v': '0.00766', 'q': '420.1435698' }] } } ''' interval = msg['ch'].split("/")[-1] for sym, updates in msg['update'].items(): symbol = self.exchange_symbol_to_std_symbol(sym) for u in updates: c = Candle(self.id, symbol, u['t'] / 1000, u['t'] / 1000 + timedelta_str_to_sec( self.normalize_candle_interval[interval]) - 0.1, self.normalize_candle_interval[interval], None, Decimal(u['o']), Decimal(u['c']), Decimal(u['h']), Decimal(u['l']), Decimal(u['v']), None, self.timestamp_normalize(u['t']), raw=msg) await self.callback(CANDLES, c, ts) async def message_handler(self, msg: str, conn, ts: float): msg = json.loads(msg, parse_float=Decimal) if 'result' in msg: LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg) elif msg['ch'] == 'orderbook/full': await self._book(msg, ts) elif msg['ch'] == 'trades': await self._trade(msg, ts) elif msg['ch'] == 'ticker/1s': await self._ticker(msg, ts) elif msg['ch'].startswith('candles/'): await self._candle(msg, ts) else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn): self.__reset() for chan in self.subscription: await conn.write( json.dumps({ "method": "subscribe", "params": { "symbols": self.subscription[chan] }, "ch": chan if chan != 'candles/' else chan + self.candle_interval_map[self.candle_interval], "id": 1234 }))
class Bitflyer(Feed): id = BITFLYER websocket_endpoints = [ WebsocketEndpoint('wss://ws.lightstream.bitflyer.com/json-rpc') ] rest_endpoints = [ RestEndpoint('https://api.bitflyer.com', routes=Routes([ '/v1/getmarkets/eu', '/v1/getmarkets/usa', '/v1/getmarkets', '/v1/markets', '/v1/markets/usa', '/v1/markets/eu' ])) ] websocket_channels = { L2_BOOK: 'lightning_board_{}', TRADES: 'lightning_executions_{}', TICKER: 'lightning_ticker_{}' } @classmethod def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for entry in data: for datum in entry: stype = datum['market_type'].lower() expiration = None if stype == FUTURES: base, quote = datum['product_code'][:3], datum[ 'product_code'][3:6] expiration = datum['product_code'][6:] elif stype == FX: _, base, quote = datum['product_code'].split("_") else: base, quote = datum['product_code'].split("_") s = Symbol(base, quote, type=stype, expiry_date=expiration) ret[s.normalized] = datum['product_code'] info['instrument_type'][s.normalized] = stype return ret, info def __reset(self): self._l2_book = {} async def _ticker(self, msg: dict, timestamp: float): """ { "jsonrpc": "2.0", "method": "channelMessage", "params": { "channel": "lightning_ticker_BTC_USD", "message": { "product_code": "BTC_USD", "state": "RUNNING", "timestamp":"2020-12-25T21:16:19.3661298Z", "tick_id": 703768, "best_bid": 24228.97, "best_ask": 24252.89, "best_bid_size": 0.4006, "best_ask_size": 0.4006, "total_bid_depth": 64.73938803, "total_ask_depth": 51.99613815, "market_bid_size": 0.0, "market_ask_size": 0.0, "ltp": 24382.25, "volume": 241.953371650000, "volume_by_product": 241.953371650000 } } } """ pair = self.exchange_symbol_to_std_symbol( msg['params']['message']['product_code']) bid = msg['params']['message']['best_bid'] ask = msg['params']['message']['best_ask'] t = Ticker(self.id, pair, bid, ask, self.timestamp_normalize( msg['params']['message']['timestamp']), raw=msg) await self.callback(TICKER, t, timestamp) async def _trade(self, msg: dict, timestamp: float): """ { "jsonrpc":"2.0", "method":"channelMessage", "params":{ "channel":"lightning_executions_BTC_JPY", "message":[ { "id":2084881071, "side":"BUY", "price":2509125.0, "size":0.005, "exec_date":"2020-12-25T21:36:22.8840579Z", "buy_child_order_acceptance_id":"JRF20201225-213620-004123", "sell_child_order_acceptance_id":"JRF20201225-213620-133314" } ] } } """ pair = self.exchange_symbol_to_std_symbol( msg['params']['channel'][21:]) for update in msg['params']['message']: t = Trade(self.id, pair, BUY if update['side'] == 'BUY' else SELL, update['size'], update['price'], self.timestamp_normalize(update['exec_date']), raw=update) await self.callback(TRADES, t, timestamp) async def _book(self, msg: dict, timestamp: float): """ { "jsonrpc":"2.0", "method":"channelMessage", "params":{ "channel":"lightning_board_BTC_JPY", "message":{ "mid_price":2534243.0, "bids":[ ], "asks":[ { "price":2534500.0, "size":0.0 }, { "price":2536101.0, "size":0.0 } ] } } } """ snapshot = msg['params']['channel'].startswith( 'lightning_board_snapshot') if snapshot: pair = msg['params']['channel'].split( "lightning_board_snapshot")[1][1:] else: pair = msg['params']['channel'].split("lightning_board")[1][1:] pair = self.exchange_symbol_to_std_symbol(pair) # Ignore deltas until a snapshot is received if pair not in self._l2_book and not snapshot: return delta = None if snapshot: self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) else: delta = {BID: [], ASK: []} data = msg['params']['message'] for side in ('bids', 'asks'): s = BID if side == 'bids' else ASK if snapshot: self._l2_book[pair].book[side] = { d['price']: d['size'] for d in data[side] } else: for entry in data[side]: if entry['size'] == 0: if entry['price'] in self._l2_book[pair].book[side]: del self._l2_book[pair].book[side][entry['price']] delta[s].append((entry['price'], Decimal(0.0))) else: self._l2_book[pair].book[side][ entry['price']] = entry['size'] delta[s].append((entry['price'], entry['size'])) await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg, delta=delta) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if msg['params']['channel'].startswith("lightning_ticker_"): await self._ticker(msg, timestamp) elif msg['params']['channel'].startswith('lightning_executions_'): await self._trade(msg, timestamp) elif msg['params']['channel'].startswith('lightning_board_'): await self._book(msg, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() for chan in self.subscription: for pair in self.subscription[chan]: if chan.startswith('lightning_board'): # need to subscribe to snapshots too if subscribed to L2_BOOKS await conn.write( json.dumps({ "method": "subscribe", "params": { "channel": f'lightning_board_snapshot_{pair}' } })) await conn.write( json.dumps({ "method": "subscribe", "params": { "channel": chan.format(pair) } }))
class Huobi(Feed): id = HUOBI websocket_endpoints = [WebsocketEndpoint('wss://api.huobi.pro/ws')] rest_endpoints = [ RestEndpoint('https://api.huobi.pro', routes=Routes('/v1/common/symbols')) ] valid_candle_intervals = { '1m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '1M', '1Y' } candle_interval_map = { '1m': '1min', '5m': '5min', '15m': '15min', '30m': '30min', '1h': '60min', '4h': '4hour', '1d': '1day', '1M': '1mon', '1w': '1week', '1Y': '1year' } websocket_channels = { L2_BOOK: 'depth.step0', TRADES: 'trade.detail', CANDLES: 'kline', TICKER: 'ticker' } @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = {'instrument_type': {}} for e in data['data']: if e['state'] == 'offline': continue base, quote = e['base-currency'].upper( ), e['quote-currency'].upper() s = Symbol(base, quote) ret[s.normalized] = e['symbol'] info['instrument_type'][s.normalized] = s.type return ret, info def __reset(self): self._l2_book = {} async def _book(self, msg: dict, timestamp: float): pair = self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1]) data = msg['tick'] if pair not in self._l2_book: self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) self._l2_book[pair].book.bids = { Decimal(price): Decimal(amount) for price, amount in data['bids'] } self._l2_book[pair].book.asks = { Decimal(price): Decimal(amount) for price, amount in data['asks'] } await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(msg['ts']), raw=msg) async def _ticker(self, msg: dict, timestamp: float): """ { "ch":"market.btcusdt.ticker", "ts":1630982370526, "tick":{ "open":51732, "high":52785.64, "low":51000, "close":52735.63, "amount":13259.24137056181, "vol":687640987.4125315, "count":448737, "bid":52732.88, "bidSize":0.036, "ask":52732.89, "askSize":0.583653, "lastPrice":52735.63, "lastSize":0.03 } } """ t = Ticker(self.id, self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1]), msg['tick']['bid'], msg['tick']['ask'], self.timestamp_normalize(msg['ts']), raw=msg['tick']) await self.callback(TICKER, t, timestamp) async def _trade(self, msg: dict, timestamp: float): """ { 'ch': 'market.adausdt.trade.detail', 'ts': 1597792835344, 'tick': { 'id': 101801945127, 'ts': 1597792835336, 'data': [ { 'id': Decimal('10180194512782291967181675'), <- per docs this is deprecated 'ts': 1597792835336, 'tradeId': 100341530602, 'amount': Decimal('0.1'), 'price': Decimal('0.137031'), 'direction': 'sell' } ] } } """ for trade in msg['tick']['data']: t = Trade(self.id, self.exchange_symbol_to_std_symbol( msg['ch'].split('.')[1]), BUY if trade['direction'] == 'buy' else SELL, Decimal(trade['amount']), Decimal(trade['price']), self.timestamp_normalize(trade['ts']), id=str(trade['tradeId']), raw=trade) await self.callback(TRADES, t, timestamp) async def _candles(self, msg: dict, symbol: str, interval: str, timestamp: float): """ { 'ch': 'market.btcusdt.kline.1min', 'ts': 1618700872863, 'tick': { 'id': 1618700820, 'open': Decimal('60751.62'), 'close': Decimal('60724.73'), 'low': Decimal('60724.73'), 'high': Decimal('60751.62'), 'amount': Decimal('2.1990737759143966'), 'vol': Decimal('133570.944386'), 'count': 235} } } """ interval = self.normalize_candle_interval[interval] start = int(msg['tick']['id']) end = start + timedelta_str_to_sec(interval) - 1 c = Candle(self.id, self.exchange_symbol_to_std_symbol(symbol), start, end, interval, msg['tick']['count'], Decimal(msg['tick']['open']), Decimal(msg['tick']['close']), Decimal(msg['tick']['high']), Decimal(msg['tick']['low']), Decimal(msg['tick']['amount']), None, self.timestamp_normalize(msg['ts']), raw=msg) await self.callback(CANDLES, c, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): # unzip message msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS) msg = json.loads(msg, parse_float=Decimal) # Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it if 'ping' in msg: await conn.write(json.dumps({'pong': msg['ping']})) elif 'status' in msg and msg['status'] == 'ok': return elif 'ch' in msg: if 'trade' in msg['ch']: await self._trade(msg, timestamp) elif 'tick' in msg['ch']: await self._ticker(msg, timestamp) elif 'depth' in msg['ch']: await self._book(msg, timestamp) elif 'kline' in msg['ch']: _, symbol, _, interval = msg['ch'].split(".") await self._candles(msg, symbol, interval, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() client_id = 0 for chan in self.subscription: for pair in self.subscription[chan]: client_id += 1 normalized_chan = self.exchange_channel_to_std(chan) await conn.write( json.dumps({ "sub": f"market.{pair}.{chan}" if normalized_chan != CANDLES else f"market.{pair}.{chan}.{self.candle_interval_map[self.candle_interval]}", "id": client_id }))
class OKX(Feed, OKXRestMixin): id = OKX_str websocket_channels = { L2_BOOK: 'books', TRADES: 'trades', TICKER: 'tickers', FUNDING: 'funding-rate', OPEN_INTEREST: 'open-interest', LIQUIDATIONS: LIQUIDATIONS, ORDER_INFO: 'orders', } websocket_endpoints = [ WebsocketEndpoint('wss://ws.okx.com:8443/ws/v5/public', channel_filter=(websocket_channels[L2_BOOK], websocket_channels[TRADES], websocket_channels[TICKER], websocket_channels[FUNDING], websocket_channels[OPEN_INTEREST], websocket_channels[LIQUIDATIONS]), options={'compression': None}), WebsocketEndpoint('wss://ws.okx.com:8443/ws/v5/private', channel_filter=(websocket_channels[ORDER_INFO], ), options={'compression': None}), ] rest_endpoints = [ RestEndpoint( 'https://www.okx.com', routes=Routes( [ '/api/v5/public/instruments?instType=SPOT', '/api/v5/public/instruments?instType=SWAP', '/api/v5/public/instruments?instType=FUTURES', '/api/v5/public/instruments?instType=OPTION&uly=BTC-USD', '/api/v5/public/instruments?instType=OPTION&uly=ETH-USD' ], liquidations= '/api/v5/public/liquidation-orders?instType={}&limit=100&state={}&uly={}' )) ] request_limit = 20 @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 @classmethod def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for entry in data: for e in entry['data']: expiry = None otype = None stype = e['instType'].lower() strike = None if stype == SPOT: base = e['baseCcy'] quote = e['quoteCcy'] elif stype == FUTURES: base, quote, expiry = e['instId'].split("-") elif stype == OPTION: base, quote, expiry, strike, otype = e['instId'].split("-") otype = PUT if otype == 'P' else CALL elif stype == 'swap': # this is a perpetual swap (aka perpetual futures contract), not a real swap stype = PERPETUAL base, quote, _ = e['instId'].split("-") s = Symbol(base, quote, expiry_date=expiry, type=stype, option_type=otype, strike_price=strike) ret[s.normalized] = e['instId'] info['tick_size'][s.normalized] = e['tickSz'] info['instrument_type'][s.normalized] = stype return ret, info async def _liquidations(self, pairs: list): last_update = defaultdict(dict) """ for PERP liquidations, the following arguments are required: uly, state for FUTURES liquidations, the following arguments are required: uly, state, alias FUTURES, MARGIN and OPTION liquidation request not currently supported by the below """ while True: for pair in pairs: if 'SWAP' in pair: instrument_type = 'SWAP' uly = pair.split("-")[0] + "-" + pair.split("-")[1] else: continue for status in (FILLED, UNFILLED): data = await self.http_conn.read( self.rest_endpoints[0].route( 'liquidations', sandbox=self.sandbox).format( instrument_type, status, uly)) data = json.loads(data, parse_float=Decimal) timestamp = time.time() if len(data['data'][0]['details']) == 0 or ( len(data['data'][0]['details']) > 0 and last_update.get(pair) == data['data'][0]['details'][0]): continue for entry in data['data'][0]['details']: if pair in last_update: if entry == last_update[pair].get(status): break liq = Liquidation( self.id, pair, BUY if entry['side'] == 'buy' else SELL, Decimal(entry['sz']), Decimal(entry['bkPx']), None, status, self.timestamp_normalize(int(entry['ts'])), raw=data) await self.callback(LIQUIDATIONS, liq, timestamp) last_update[pair][status] = data['data'][0]['details'][0] await asyncio.sleep(0.1) await asyncio.sleep(60) def __reset(self): self._l2_book = {} @classmethod def instrument_type(cls, symbol: str): return cls.info()['instrument_type'][symbol] async def _ticker(self, msg: dict, timestamp: float): """ {"arg": {"channel": "tickers", "instId": "LTC-USD-200327"}, "data": [{"instType": "SWAP","instId": "LTC-USD-SWAP","last": "9999.99","lastSz": "0.1","askPx": "9999.99","askSz": "11","bidPx": "8888.88","bidSz": "5","open24h": "9000","high24h": "10000","low24h": "8888.88","volCcy24h": "2222","vol24h": "2222","sodUtc0": "2222","sodUtc8": "2222","ts": "1597026383085"}]} """ pair = self.exchange_symbol_to_std_symbol(msg['arg']['instId']) for update in msg['data']: update_timestamp = self.timestamp_normalize(int(update['ts'])) t = Ticker( self.id, pair, Decimal(update['bidPx']) if update['bidPx'] else Decimal(0), Decimal(update['askPx']) if update['askPx'] else Decimal(0), update_timestamp, raw=update) await self.callback(TICKER, t, timestamp) async def _open_interest(self, msg: dict, timestamp: float): """ { 'arg': { 'channel': 'open-interest', 'instId': 'BTC-USDT-SWAP }, 'data': [ { 'instId': 'BTC-USDT-SWAP', 'instType': 'SWAP', 'oi':'565474', 'oiCcy': '5654.74', 'ts': '1630338003010' } ] } """ symbol = self.exchange_symbol_to_std_symbol(msg['arg']['instId']) for update in msg['data']: oi = OpenInterest(self.id, symbol, Decimal(update['oi']), self.timestamp_normalize(int(update['ts'])), raw=update) await self.callback(OPEN_INTEREST, oi, timestamp) async def _trade(self, msg: dict, timestamp: float): """ { "arg": { "channel": "trades", "instId": "BTC-USD-191227" }, "data": [ { "instId": "BTC-USD-191227", "tradeId": "9", "px": "0.016", "sz": "50", "side": "buy", "ts": "1597026383085" } ] } """ for trade in msg['data']: t = Trade(self.id, self.exchange_symbol_to_std_symbol(trade['instId']), BUY if trade['side'] == 'buy' else SELL, Decimal(trade['sz']), Decimal(trade['px']), self.timestamp_normalize(int(trade['ts'])), id=trade['tradeId'], raw=trade) await self.callback(TRADES, t, timestamp) async def _funding(self, msg: dict, timestamp: float): for update in msg['data']: f = Funding(self.id, self.exchange_symbol_to_std_symbol(update['instId']), None, Decimal(update['fundingRate']), None, self.timestamp_normalize(int(update['fundingTime'])), predicted_rate=Decimal(update['nextFundingRate']), raw=update) await self.callback(FUNDING, f, timestamp) async def _book(self, msg: dict, timestamp: float): if msg['action'] == 'snapshot': # snapshot pair = self.exchange_symbol_to_std_symbol(msg['arg']['instId']) for update in msg['data']: bids = { Decimal(price): Decimal(amount) for price, amount, *_ in update['bids'] } asks = { Decimal(price): Decimal(amount) for price, amount, *_ in update['asks'] } self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, checksum_format=self.id, bids=bids, asks=asks) if self.checksum_validation and self._l2_book[ pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF): raise BadChecksum await self.book_callback( L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(int(update['ts'])), checksum=update['checksum'] & 0xFFFFFFFF, raw=msg) else: # update pair = self.exchange_symbol_to_std_symbol(msg['arg']['instId']) for update in msg['data']: delta = {BID: [], ASK: []} for side in ('bids', 'asks'): s = BID if side == 'bids' else ASK for price, amount, *_ in update[side]: price = Decimal(price) amount = Decimal(amount) if amount == 0: if price in self._l2_book[pair].book[s]: delta[s].append((price, 0)) del self._l2_book[pair].book[s][price] else: delta[s].append((price, amount)) self._l2_book[pair].book[s][price] = amount if self.checksum_validation and self._l2_book[ pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF): raise BadChecksum await self.book_callback( L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(int(update['ts'])), raw=msg, delta=delta, checksum=update['checksum'] & 0xFFFFFFFF) async def _order(self, msg: dict, timestamp: float): ''' { "arg": { "channel": "orders", "instType": "FUTURES", "instId": "BTC-USD-200329" }, "data": [ { "instType": "FUTURES", "instId": "BTC-USD-200329", "ccy": "BTC", "ordId": "312269865356374016", "clOrdId": "b1", "tag": "", "px": "999", "sz": "333", "notionalUsd": "", "ordType": "limit", "side": "buy", "posSide": "long", "tdMode": "cross", "tgtCcy": "", "fillSz": "0", "fillPx": "long", "tradeId": "0", "accFillSz": "323", "fillNotionalUsd": "", "fillTime": "0", "fillFee": "0.0001", "fillFeeCcy": "BTC", "execType": "T", "state": "canceled", "avgPx": "0", "lever": "20", "tpTriggerPx": "0", "tpOrdPx": "20", "slTriggerPx": "0", "slOrdPx": "20", "feeCcy": "", "fee": "", "rebateCcy": "", "rebate": "", "tgtCcy":"", "pnl": "", "category": "", "uTime": "1597026383085", "cTime": "1597026383085", "reqId": "", "amendResult": "", "code": "0", "msg": "" } ] } ''' status = msg['data'][0]['state'] if status == 'canceled': status == CANCELLED elif status == 'live': status == OPEN elif status == 'partially-filled': status = PARTIAL elif status == 'filled': status = FILLED o_type = msg['data'][0]['ordType'] if o_type == 'market': o_type = MARKET elif o_type == 'post_only': o_type = MAKER_OR_CANCEL elif o_type == 'fok': o_type = FILL_OR_KILL elif o_type == 'ioc': o_type = IMMEDIATE_OR_CANCEL elif o_type == 'limit': o_type = LIMIT oi = OrderInfo( self.id, self.exchange_symbol_to_std_symbol( msg['data'][0]['instId'].upper()), msg['data'][0]['ordId'], BUY if msg['data'][0]['side'].lower() == 'buy' else SELL, status, o_type, Decimal(msg['data'][0]['px']) if msg['data'][0]['px'] else Decimal(msg['data'][0]['avgPx']), Decimal(msg['data'][0]['sz']), Decimal(msg['data'][0]['sz']) - Decimal(msg['data'][0]['accFillSz']) if msg['data'][0]['accFillSz'] else Decimal(msg['data'][0]['sz']), self.timestamp_normalize(int(msg['data'][0]['uTime'])), raw=msg) await self.callback(ORDER_INFO, oi, timestamp) async def _login(self, msg: dict, timestamp: float): LOG.debug('%s: Websocket logged in? %s', self.id, msg['code']) async def message_handler(self, msg: str, conn, timestamp: float): # DEFLATE compression, no header # msg = zlib.decompress(msg, -15) # not required, as websocket now set to "Per-Message Deflate" msg = json.loads(msg, parse_float=Decimal) if 'event' in msg: if msg['event'] == 'error': LOG.error("%s: Error: %s", self.id, msg) elif msg['event'] == 'subscribe': pass elif msg['event'] == 'login': await self._login(msg, timestamp) else: LOG.warning("%s: Unhandled event %s", self.id, msg) elif 'arg' in msg: if self.websocket_channels[L2_BOOK] in msg['arg']['channel']: await self._book(msg, timestamp) elif self.websocket_channels[TICKER] in msg['arg']['channel']: await self._ticker(msg, timestamp) elif self.websocket_channels[TRADES] in msg['arg']['channel']: await self._trade(msg, timestamp) elif self.websocket_channels[FUNDING] in msg['arg']['channel']: await self._funding(msg, timestamp) elif self.websocket_channels[ORDER_INFO] in msg['arg']['channel']: await self._order(msg, timestamp) elif self.websocket_channels[OPEN_INTEREST] in msg['arg'][ 'channel']: await self._open_interest(msg, timestamp) else: LOG.warning("%s: Unhandled message %s", self.id, msg) async def subscribe(self, connection: AsyncConnection): channels = [] for chan in self.subscription: if chan == LIQUIDATIONS: asyncio.create_task(self._liquidations( self.subscription[chan])) continue for pair in self.subscription[chan]: channels.append(self.build_subscription(chan, pair)) msg = {"op": "subscribe", "args": channels} await connection.write(json.dumps(msg)) async def authenticate(self, conn: AsyncConnection): if self.requires_authentication: if any([ self.is_authenticated_channel( self.exchange_channel_to_std(chan)) for chan in conn.subscription ]): auth = self._auth(self.key_id, self.key_secret) LOG.debug(f"{conn.uuid}: Authenticating with message: {auth}") await conn.write(json.dumps(auth)) await asyncio.sleep(1) def _auth(self, key_id, key_secret) -> str: timestamp, sign = self._generate_token(key_id, key_secret) login_param = { "op": "login", "args": [{ "apiKey": self.key_id, "passphrase": self.key_passphrase, "timestamp": timestamp, "sign": sign.decode("utf-8") }] } return login_param def build_subscription(self, channel: str, ticker: str) -> dict: if channel in ['positions', 'orders']: subscription_dict = { "channel": channel, "instType": self.inst_type_to_okx_type(ticker), "instId": ticker } else: subscription_dict = {"channel": channel, "instId": ticker} return subscription_dict def inst_type_to_okx_type(self, ticker): sym = self.exchange_symbol_to_std_symbol(ticker) instrument_type = self.instrument_type(sym) instrument_type_map = { 'perpetual': 'SWAP', 'spot': 'MARGIN', 'futures': 'FUTURES', 'option': 'OPTION' } return instrument_type_map.get(instrument_type, 'MARGIN') def _get_server_time(self): endpoint = "public/time" response = requests.get(self.api + endpoint) if response.status_code == 200: return response.json()['data'][0]['ts'] else: return "" def _server_timestamp(self): server_time = self._get_server_time() return int(server_time) / 1000 def _create_sign(self, timestamp: str, key_secret: str): message = timestamp + 'GET' + '/users/self/verify' mac = hmac.new(bytes(key_secret, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256') d = mac.digest() sign = base64.b64encode(d) return sign def _generate_token(self, key_id: str, key_secret: str) -> dict: timestamp = str(self._server_timestamp()) sign = self._create_sign(timestamp, key_secret) return timestamp, sign
class Bybit(Feed): id = BYBIT websocket_channels = { # TODO make aggregation window (100ms) configurable # TODO figure out ValueError: Use subscription, or channels and symbols, not both # TODO OPEN_INTEREST and FUNDING report INDEX data_type L2_BOOK: 'orderBook_200.100ms', TRADES: 'trade', FILLS: 'execution', ORDER_INFO: 'order', INDEX: 'instrument_info.100ms', OPEN_INTEREST: 'instrument_info.100ms', FUNDING: 'instrument_info.100ms', CANDLES: 'klineV2', LIQUIDATIONS: 'liquidation' } websocket_endpoints = [ WebsocketEndpoint('wss://stream.bybit.com/realtime', channel_filter=(websocket_channels[L2_BOOK], websocket_channels[TRADES], websocket_channels[INDEX], websocket_channels[OPEN_INTEREST], websocket_channels[FUNDING], websocket_channels[CANDLES], websocket_channels[LIQUIDATIONS]), instrument_filter=('QUOTE', ('USD',)), sandbox='wss://stream-testnet.bybit.com/realtime', options={'compression': None}), WebsocketEndpoint('wss://stream.bybit.com/realtime_public', channel_filter=(websocket_channels[L2_BOOK], websocket_channels[TRADES], websocket_channels[INDEX], websocket_channels[OPEN_INTEREST], websocket_channels[FUNDING], websocket_channels[CANDLES], websocket_channels[LIQUIDATIONS]), instrument_filter=('QUOTE', ('USDT',)), sandbox='wss://stream-testnet.bybit.com/realtime_public', options={'compression': None}), WebsocketEndpoint('wss://stream.bybit.com/realtime_private', channel_filter=(websocket_channels[ORDER_INFO], websocket_channels[FILLS]), instrument_filter=('QUOTE', ('USDT',)), sandbox='wss://stream-testnet.bybit.com/realtime_private', options={'compression': None}), ] rest_endpoints = [RestEndpoint('https://api.bybit.com', routes=Routes('/v2/public/symbols'))] valid_candle_intervals = {'1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '1d', '1w', '1M'} candle_interval_map = {'1m': '1', '3m': '3', '5m': '5', '15m': '15', '30m': '30', '1h': '60', '2h': '120', '4h': '240', '6h': '360', '1d': 'D', '1w': 'W', '1M': 'M'} @classmethod def timestamp_normalize(cls, ts: Union[int, dt]) -> float: if isinstance(ts, int): return ts / 1000.0 else: return ts.timestamp() @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) # PERPETUAL & FUTURES for symbol in data['result']: base = symbol['base_currency'] quote = symbol['quote_currency'] stype = PERPETUAL expiry = None if not symbol['name'].endswith(quote): stype = FUTURES year = symbol['name'].replace(base + quote, '')[-2:] expiry = year + symbol['alias'].replace(base + quote, '')[-4:] s = Symbol(base, quote, type=stype, expiry_date=expiry) ret[s.normalized] = symbol['name'] info['tick_size'][s.normalized] = symbol['price_filter']['tick_size'] info['instrument_type'][s.normalized] = stype return ret, info def __reset(self): self._instrument_info_cache = {} self._l2_book = {} async def _candle(self, msg: dict, timestamp: float): ''' { "topic": "klineV2.1.BTCUSD", //topic name "data": [{ "start": 1572425640, //start time of the candle "end": 1572425700, //end time of the candle "open": 9200, //open price "close": 9202.5, //close price "high": 9202.5, //max price "low": 9196, //min price "volume": 81790, //volume "turnover": 8.889247899999999, //turnover "confirm": False, //snapshot flag (indicates if candle is closed or not) "cross_seq": 297503466, "timestamp": 1572425676958323 //cross time }], "timestamp_e6": 1572425677047994 //server time } ''' symbol = self.exchange_symbol_to_std_symbol(msg['topic'].split(".")[-1]) ts = msg['timestamp_e6'] / 1_000_000 for entry in msg['data']: if self.candle_closed_only and not entry['confirm']: continue c = Candle(self.id, symbol, entry['start'], entry['end'], self.candle_interval, entry['confirm'], Decimal(entry['open']), Decimal(entry['close']), Decimal(entry['high']), Decimal(entry['low']), Decimal(entry['volume']), None, ts, raw=entry) await self.callback(CANDLES, c, timestamp) async def _liquidation(self, msg: dict, timestamp: float): ''' { 'topic': 'liquidation.EOSUSDT', 'data': { 'symbol': 'EOSUSDT', 'side': 'Buy', 'price': '3.950', 'qty': '58.0', 'time': 1632586154956 } } ''' liq = Liquidation( self.id, self.exchange_symbol_to_std_symbol(msg['data']['symbol']), BUY if msg['data']['side'] == 'Buy' else SELL, Decimal(msg['data']['qty']), Decimal(msg['data']['price']), None, None, self.timestamp_normalize(msg['data']['time']), raw=msg ) await self.callback(LIQUIDATIONS, liq, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if "success" in msg: if msg['success']: if msg['request']['op'] == 'auth': LOG.debug("%s: Authenticated successful", conn.uuid) elif msg['request']['op'] == 'subscribe': LOG.debug("%s: Subscribed to channels: %s", conn.uuid, msg['request']['args']) else: LOG.warning("%s: Unhandled 'successs' message received", conn.uuid) else: LOG.error("%s: Error from exchange %s", conn.uuid, msg) elif msg["topic"].startswith('trade'): await self._trade(msg, timestamp) elif msg["topic"].startswith('orderBook'): await self._book(msg, timestamp) elif msg['topic'].startswith('liquidation'): await self._liquidation(msg, timestamp) elif "instrument_info" in msg["topic"]: await self._instrument_info(msg, timestamp) elif "order" in msg["topic"]: await self._order(msg, timestamp) elif "execution" in msg["topic"]: await self._execution(msg, timestamp) elif 'klineV2' in msg['topic'] or 'candle' in msg['topic']: await self._candle(msg, timestamp) # elif "position" in msg["topic"]: # await self._balances(msg, timestamp) else: LOG.warning("%s: Unhandled message type %s", conn.uuid, msg) async def subscribe(self, connection: AsyncConnection): self.__reset() for chan in connection.subscription: if not self.is_authenticated_channel(self.exchange_channel_to_std(chan)): for pair in connection.subscription[chan]: sym = str_to_symbol(self.exchange_symbol_to_std_symbol(pair)) if self.exchange_channel_to_std(chan) == CANDLES: c = chan if sym.quote == 'USD' else 'candle' sub = [f"{c}.{self.candle_interval_map[self.candle_interval]}.{pair}"] else: sub = [f"{chan}.{pair}"] await connection.write(json.dumps({"op": "subscribe", "args": sub})) else: await connection.write(json.dumps( { "op": "subscribe", "args": [f"{chan}"] } )) async def _instrument_info(self, msg: dict, timestamp: float): """ ### Snapshot type update { "topic": "instrument_info.100ms.BTCUSD", "type": "snapshot", "data": { "id": 1, "symbol": "BTCUSD", //instrument name "last_price_e4": 81165000, //the latest price "last_tick_direction": "ZeroPlusTick", //the direction of last tick:PlusTick,ZeroPlusTick,MinusTick, //ZeroMinusTick "prev_price_24h_e4": 81585000, //the price of prev 24h "price_24h_pcnt_e6": -5148, //the current last price percentage change from prev 24h price "high_price_24h_e4": 82900000, //the highest price of prev 24h "low_price_24h_e4": 79655000, //the lowest price of prev 24h "prev_price_1h_e4": 81395000, //the price of prev 1h "price_1h_pcnt_e6": -2825, //the current last price percentage change from prev 1h price "mark_price_e4": 81178500, //mark price "index_price_e4": 81172800, //index price "open_interest": 154418471, //open interest quantity - Attention, the update is not //immediate - slowest update is 1 minute "open_value_e8": 1997561103030, //open value quantity - Attention, the update is not //immediate - the slowest update is 1 minute "total_turnover_e8": 2029370141961401, //total turnover "turnover_24h_e8": 9072939873591, //24h turnover "total_volume": 175654418740, //total volume "volume_24h": 735865248, //24h volume "funding_rate_e6": 100, //funding rate "predicted_funding_rate_e6": 100, //predicted funding rate "cross_seq": 1053192577, //sequence "created_at": "2018-11-14T16:33:26Z", "updated_at": "2020-01-12T18:25:16Z", "next_funding_time": "2020-01-13T00:00:00Z", //next funding time //the rest time to settle funding fee "countdown_hour": 6 //the remaining time to settle the funding fee }, "cross_seq": 1053192634, "timestamp_e6": 1578853524091081 //the timestamp when this information was produced } ### Delta type update { "topic": "instrument_info.100ms.BTCUSD", "type": "delta", "data": { "delete": [], "update": [ { "id": 1, "symbol": "BTCUSD", "prev_price_24h_e4": 81565000, "price_24h_pcnt_e6": -4904, "open_value_e8": 2000479681106, "total_turnover_e8": 2029370495672976, "turnover_24h_e8": 9066215468687, "volume_24h": 735316391, "cross_seq": 1053192657, "created_at": "2018-11-14T16:33:26Z", "updated_at": "2020-01-12T18:25:25Z" } ], "insert": [] }, "cross_seq": 1053192657, "timestamp_e6": 1578853525691123 } """ update_type = msg['type'] if update_type == 'snapshot': updates = [msg['data']] else: updates = msg['data']['update'] for info in updates: if msg['topic'] in self._instrument_info_cache and self._instrument_info_cache[msg['topic']] == updates: continue else: self._instrument_info_cache[msg['topic']] = updates ts = int(msg['timestamp_e6']) / 1_000_000 if 'open_interest' in info: oi = OpenInterest( self.id, self.exchange_symbol_to_std_symbol(info['symbol']), Decimal(info['open_interest']), ts, raw=info ) await self.callback(OPEN_INTEREST, oi, timestamp) if 'index_price_e4' in info: i = Index( self.id, self.exchange_symbol_to_std_symbol(info['symbol']), Decimal(info['index_price_e4']) * Decimal('1e-4'), ts, raw=info ) await self.callback(INDEX, i, timestamp) if 'funding_rate_e6' in info: f = Funding( self.id, self.exchange_symbol_to_std_symbol(info['symbol']), None, Decimal(info['funding_rate_e6']) * Decimal('1e-6'), info['next_funding_time'].timestamp(), ts, predicted_rate=Decimal(info['predicted_funding_rate_e6']) * Decimal('1e-6'), raw=info ) await self.callback(FUNDING, f, timestamp) async def _trade(self, msg: dict, timestamp: float): """ {"topic":"trade.BTCUSD", "data":[ { "timestamp":"2019-01-22T15:04:33.461Z", "symbol":"BTCUSD", "side":"Buy", "size":980, "price":3563.5, "tick_direction":"PlusTick", "trade_id":"9d229f26-09a8-42f8-aff3-0ba047b0449d", "cross_seq":163261271}]} """ data = msg['data'] for trade in data: if isinstance(trade['trade_time_ms'], str): ts = int(trade['trade_time_ms']) else: ts = trade['trade_time_ms'] t = Trade( self.id, self.exchange_symbol_to_std_symbol(trade['symbol']), BUY if trade['side'] == 'Buy' else SELL, Decimal(trade['size']), Decimal(trade['price']), self.timestamp_normalize(ts), id=trade['trade_id'], raw=trade ) await self.callback(TRADES, t, timestamp) async def _book(self, msg: dict, timestamp: float): pair = self.exchange_symbol_to_std_symbol(msg['topic'].split('.')[-1]) update_type = msg['type'] data = msg['data'] delta = {BID: [], ASK: []} if update_type == 'snapshot': delta = None self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) # the USDT perpetual data is under the order_book key if 'order_book' in data: data = data['order_book'] for update in data: side = BID if update['side'] == 'Buy' else ASK self._l2_book[pair].book[side][Decimal(update['price'])] = Decimal(update['size']) else: for delete in data['delete']: side = BID if delete['side'] == 'Buy' else ASK price = Decimal(delete['price']) delta[side].append((price, 0)) del self._l2_book[pair].book[side][price] for utype in ('update', 'insert'): for update in data[utype]: side = BID if update['side'] == 'Buy' else ASK price = Decimal(update['price']) amount = Decimal(update['size']) delta[side].append((price, amount)) self._l2_book[pair].book[side][price] = amount # timestamp is in microseconds ts = msg['timestamp_e6'] if isinstance(ts, str): ts = int(ts) await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=ts / 1000000, raw=msg, delta=delta) async def _order(self, msg: dict, timestamp: float): """ { "topic": "order", "action": "", "data": [ { "order_id": "xxxxxxxx-xxxx-xxxx-9a8f-4a973eb5c418", "order_link_id": "", "symbol": "BTCUSDT", "side": "Buy", "order_type": "Limit", "price": 11000, "qty": 0.001, "leaves_qty": 0.001, "last_exec_price": 0, "cum_exec_qty": 0, "cum_exec_value": 0, "cum_exec_fee": 0, "time_in_force": "GoodTillCancel", "create_type": "CreateByUser", "cancel_type": "UNKNOWN", "order_status": "New", "take_profit": 0, "stop_loss": 0, "trailing_stop": 0, "reduce_only": false, "close_on_trigger": false, "create_time": "2020-08-12T21:18:40.780039678Z", "update_time": "2020-08-12T21:18:40.787986415Z" } ] } """ order_status = { 'Created': SUBMITTING, 'Rejected': FAILED, 'New': OPEN, 'PartiallyFilled': PARTIAL, 'Filled': FILLED, 'Cancelled': CANCELLED, 'PendingCancel': CANCELLING } for i in range(len(msg['data'])): data = msg['data'][i] oi = OrderInfo( self.id, self.exchange_symbol_to_std_symbol(data['symbol']), data["order_id"], BUY if data["side"] == 'Buy' else SELL, order_status[data["order_status"]], LIMIT if data['order_type'] == 'Limit' else MARKET, Decimal(data['price']), Decimal(data['cum_exec_qty']), Decimal(data['qty']) - Decimal(data['cum_exec_qty']), self.timestamp_normalize(data.get('update_time') or data.get('O') or data.get('timestamp')), raw=data, ) await self.callback(ORDER_INFO, oi, timestamp) async def _execution(self, msg: dict, timestamp: float): ''' { "topic": "execution", "data": [ { "symbol": "BTCUSD", "side": "Buy", "order_id": "xxxxxxxx-xxxx-xxxx-9a8f-4a973eb5c418", "exec_id": "xxxxxxxx-xxxx-xxxx-8b66-c3d2fcd352f6", "order_link_id": "", "price": "8300", "order_qty": 1, "exec_type": "Trade", "exec_qty": 1, "exec_fee": "0.00000009", "leaves_qty": 0, "is_maker": false, "trade_time": "2020-01-14T14:07:23.629Z" // trade time } ] } ''' for entry in msg['data']: symbol = self.exchange_symbol_to_std_symbol(entry['symbol']) f = Fill( self.id, symbol, BUY if entry['side'] == 'Buy' else SELL, Decimal(entry['exec_qty']), Decimal(entry['price']), Decimal(entry['exec_fee']), entry['exec_id'], entry['order_id'], None, MAKER if entry['is_maker'] else TAKER, entry['trade_time'].timestamp(), raw=entry ) await self.callback(FILLS, f, timestamp) # async def _balances(self, msg: dict, timestamp: float): # for i in range(len(msg['data'])): # data = msg['data'][i] # symbol = self.exchange_symbol_to_std_symbol(data['symbol']) # await self.callback(BALANCES, feed=self.id, symbol=symbol, data=data, receipt_timestamp=timestamp) async def authenticate(self, conn: AsyncConnection): if any(self.is_authenticated_channel(self.exchange_channel_to_std(chan)) for chan in conn.subscription): auth = self._auth(self.key_id, self.key_secret) LOG.debug(f"{conn.uuid}: Sending authentication request with message {auth}") await conn.write(auth) def _auth(self, key_id: str, key_secret: str) -> str: # https://bybit-exchange.github.io/docs/inverse/#t-websocketauthentication expires = int((time.time() + 60)) * 1000 signature = str(hmac.new(bytes(key_secret, 'utf-8'), bytes(f'GET/realtime{expires}', 'utf-8'), digestmod='sha256').hexdigest()) return json.dumps({'op': 'auth', 'args': [key_id, expires, signature]})
class Binance(Feed, BinanceRestMixin): id = BINANCE websocket_endpoints = [ WebsocketEndpoint('wss://stream.binance.com:9443', sandbox='wss://testnet.binance.vision') ] rest_endpoints = [ RestEndpoint('https://api.binance.com', routes=Routes('/api/v3/exchangeInfo', l2book='/api/v3/depth?symbol={}&limit={}', authentication='/api/v3/userDataStream'), sandbox='https://testnet.binance.vision') ] valid_depths = [5, 10, 20, 50, 100, 500, 1000, 5000] # m -> minutes; h -> hours; d -> days; w -> weeks; M -> months valid_candle_intervals = { '1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w', '1M' } valid_depth_intervals = {'100ms', '1000ms'} websocket_channels = { L2_BOOK: 'depth', TRADES: 'aggTrade', TICKER: 'bookTicker', CANDLES: 'kline_', BALANCES: BALANCES, ORDER_INFO: ORDER_INFO } request_limit = 20 @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for symbol in data['symbols']: if symbol.get('status', 'TRADING') != "TRADING": continue if symbol.get('contractStatus', 'TRADING') != "TRADING": continue expiration = None stype = SPOT if symbol.get('contractType') == 'PERPETUAL': stype = PERPETUAL elif symbol.get('contractType') in ('CURRENT_QUARTER', 'NEXT_QUARTER'): stype = FUTURES expiration = symbol['symbol'].split("_")[1] s = Symbol(symbol['baseAsset'], symbol['quoteAsset'], type=stype, expiry_date=expiration) ret[s.normalized] = symbol['symbol'] info['tick_size'][s.normalized] = symbol['filters'][0]['tickSize'] info['instrument_type'][s.normalized] = stype return ret, info def __init__(self, depth_interval='100ms', **kwargs): """ depth_interval: str time between l2_book/delta updates {'100ms', '1000ms'} (different from BINANCE_FUTURES & BINANCE_DELIVERY) """ if depth_interval is not None and depth_interval not in self.valid_depth_intervals: raise ValueError( f"Depth interval must be one of {self.valid_depth_intervals}") super().__init__(**kwargs) self.depth_interval = depth_interval self._open_interest_cache = {} self._reset() def _address(self) -> Union[str, Dict]: """ Binance has a 200 pair/stream limit per connection, so we need to break the address down into multiple connections if necessary. Because the key is currently not used for the address dict, we can just set it to the last used stream, since this will be unique. The generic connect method supplied by Feed will take care of creating the correct connection objects from the addresses. """ if self.requires_authentication: listen_key = self._generate_token() address = self.address address += '/ws/' + listen_key else: address = self.address address += '/stream?streams=' subs = [] is_any_private = any( self.is_authenticated_channel(chan) for chan in self.subscription) is_any_public = any(not self.is_authenticated_channel(chan) for chan in self.subscription) if is_any_private and is_any_public: raise ValueError( "Private channels should be subscribed in separate feeds vs public channels" ) if all( self.is_authenticated_channel(chan) for chan in self.subscription): return address for chan in self.subscription: normalized_chan = self.exchange_channel_to_std(chan) if normalized_chan == OPEN_INTEREST: continue if self.is_authenticated_channel(normalized_chan): continue stream = chan if normalized_chan == CANDLES: stream = f"{chan}{self.candle_interval}" elif normalized_chan == L2_BOOK: stream = f"{chan}@{self.depth_interval}" for pair in self.subscription[chan]: # for everything but premium index the symbols need to be lowercase. if pair.startswith("p"): if normalized_chan != CANDLES: raise ValueError( "Premium Index Symbols only allowed on Candle data feed" ) else: pair = pair.lower() subs.append(f"{pair}@{stream}") if 0 < len(subs) < 200: return address + '/'.join(subs) else: def split_list(_list: list, n: int): for i in range(0, len(_list), n): yield _list[i:i + n] return [ address + '/'.join(chunk) for chunk in split_list(subs, 200) ] def _reset(self): self._l2_book = {} self.last_update_id = {} async def _refresh_token(self): while True: await sleep(30 * 60) if self._auth_token is None: raise ValueError('There is no token to refresh') payload = {'listenKey': self._auth_token} r = requests.put( f'{self.rest_endpoints[0].route("authentication", sandbox=self.sandbox)}?{urlencode(payload)}', headers={'X-MBX-APIKEY': self.key_id}) r.raise_for_status() def _generate_token(self) -> str: url = self.rest_endpoints[0].route('authentication', sandbox=self.sandbox) r = requests.post(url, headers={'X-MBX-APIKEY': self.key_id}) r.raise_for_status() response = r.json() if 'listenKey' in response: self._auth_token = response['listenKey'] return self._auth_token else: raise ValueError(f'Unable to retrieve listenKey token from {url}') async def _trade(self, msg: dict, timestamp: float): """ { "e": "aggTrade", // Event type "E": 123456789, // Event time "s": "BNBBTC", // Symbol "a": 12345, // Aggregate trade ID "p": "0.001", // Price "q": "100", // Quantity "f": 100, // First trade ID "l": 105, // Last trade ID "T": 123456785, // Trade time "m": true, // Is the buyer the market maker? "M": true // Ignore } """ t = Trade(self.id, self.exchange_symbol_to_std_symbol(msg['s']), SELL if msg['m'] else BUY, Decimal(msg['q']), Decimal(msg['p']), self.timestamp_normalize(msg['T']), id=str(msg['a']), raw=msg) await self.callback(TRADES, t, timestamp) async def _ticker(self, msg: dict, timestamp: float): """ { 'u': 382569232, 's': 'FETUSDT', 'b': '0.36031000', 'B': '1500.00000000', 'a': '0.36092000', 'A': '176.40000000' } """ pair = self.exchange_symbol_to_std_symbol(msg['s']) bid = Decimal(msg['b']) ask = Decimal(msg['a']) # Binance does not have a timestamp in this update, but the two futures APIs do if 'E' in msg: ts = self.timestamp_normalize(msg['E']) else: ts = timestamp t = Ticker(self.id, pair, bid, ask, ts, raw=msg) await self.callback(TICKER, t, timestamp) async def _liquidations(self, msg: dict, timestamp: float): """ { "e":"forceOrder", // Event Type "E":1568014460893, // Event Time "o":{ "s":"BTCUSDT", // Symbol "S":"SELL", // Side "o":"LIMIT", // Order Type "f":"IOC", // Time in Force "q":"0.014", // Original Quantity "p":"9910", // Price "ap":"9910", // Average Price "X":"FILLED", // Order Status "l":"0.014", // Order Last Filled Quantity "z":"0.014", // Order Filled Accumulated Quantity "T":1568014460893, // Order Trade Time } } """ pair = self.exchange_symbol_to_std_symbol(msg['o']['s']) liq = Liquidation(self.id, pair, SELL if msg['o']['S'] == 'SELL' else BUY, Decimal(msg['o']['q']), Decimal(msg['o']['p']), None, FILLED if msg['o']['X'] == 'FILLED' else UNFILLED, self.timestamp_normalize(msg['E']), raw=msg) await self.callback(LIQUIDATIONS, liq, receipt_timestamp=timestamp) def _check_update_id(self, std_pair: str, msg: dict) -> bool: """ Messages will be queued while fetching snapshot and we can return a book_callback using this msg's data instead of waiting for the next update. """ if self._l2_book[std_pair].delta is None and msg[ 'u'] <= self.last_update_id[std_pair]: return True elif msg['U'] <= self.last_update_id[std_pair] and msg[ 'u'] <= self.last_update_id[std_pair]: # Old message, can ignore it return True elif msg['U'] <= self.last_update_id[std_pair] + 1 <= msg['u']: self.last_update_id[std_pair] = msg['u'] return False elif self.last_update_id[std_pair] + 1 == msg['U']: self.last_update_id[std_pair] = msg['u'] return False else: self._reset() LOG.warning("%s: Missing book update detected, resetting book", self.id) return True async def _snapshot(self, pair: str) -> None: max_depth = self.max_depth if self.max_depth else 1000 if max_depth not in self.valid_depths: for d in self.valid_depths: if d > max_depth: max_depth = d break resp = await self.http_conn.read(self.rest_endpoints[0].route( 'l2book', self.sandbox).format(pair, max_depth)) resp = json.loads(resp, parse_float=Decimal) timestamp = self.timestamp_normalize( resp['E']) if 'E' in resp else None std_pair = self.exchange_symbol_to_std_symbol(pair) self.last_update_id[std_pair] = resp['lastUpdateId'] self._l2_book[std_pair] = OrderBook( self.id, std_pair, max_depth=self.max_depth, bids={Decimal(u[0]): Decimal(u[1]) for u in resp['bids']}, asks={Decimal(u[0]): Decimal(u[1]) for u in resp['asks']}) await self.book_callback(L2_BOOK, self._l2_book[std_pair], time.time(), timestamp=timestamp, raw=resp, sequence_number=self.last_update_id[std_pair]) async def _book(self, msg: dict, pair: str, timestamp: float): """ { "e": "depthUpdate", // Event type "E": 123456789, // Event time "s": "BNBBTC", // Symbol "U": 157, // First update ID in event "u": 160, // Final update ID in event "b": [ // Bids to be updated [ "0.0024", // Price level to be updated "10" // Quantity ] ], "a": [ // Asks to be updated [ "0.0026", // Price level to be updated "100" // Quantity ] ] } """ exchange_pair = pair pair = self.exchange_symbol_to_std_symbol(pair) if pair not in self._l2_book: await self._snapshot(exchange_pair) skip_update = self._check_update_id(pair, msg) if skip_update: return delta = {BID: [], ASK: []} for s, side in (('b', BID), ('a', ASK)): for update in msg[s]: price = Decimal(update[0]) amount = Decimal(update[1]) delta[side].append((price, amount)) if amount == 0: if price in self._l2_book[pair].book[side]: del self._l2_book[pair].book[side][price] else: self._l2_book[pair].book[side][price] = amount await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(msg['E']), raw=msg, delta=delta, sequence_number=self.last_update_id[pair]) async def _funding(self, msg: dict, timestamp: float): """ { "e": "markPriceUpdate", // Event type "E": 1562305380000, // Event time "s": "BTCUSDT", // Symbol "p": "11185.87786614", // Mark price "r": "0.00030000", // Funding rate "T": 1562306400000 // Next funding time } BinanceFutures { "e": "markPriceUpdate", // Event type "E": 1562305380000, // Event time "s": "BTCUSDT", // Symbol "p": "11185.87786614", // Mark price "i": "11784.62659091" // Index price "P": "11784.25641265", // Estimated Settle Price, only useful in the last hour before the settlement starts "r": "0.00030000", // Funding rate "T": 1562306400000 // Next funding time } """ next_time = self.timestamp_normalize( msg['T']) if msg['T'] > 0 else None rate = Decimal(msg['r']) if msg['r'] else None if next_time is None: rate = None f = Funding(self.id, self.exchange_symbol_to_std_symbol(msg['s']), Decimal(msg['p']), rate, next_time, self.timestamp_normalize(msg['E']), predicted_rate=Decimal(msg['P']) if 'P' in msg and msg['P'] is not None else None, raw=msg) await self.callback(FUNDING, f, timestamp) async def _candle(self, msg: dict, timestamp: float): """ { 'e': 'kline', 'E': 1615927655524, 's': 'BTCUSDT', 'k': { 't': 1615927620000, 'T': 1615927679999, 's': 'BTCUSDT', 'i': '1m', 'f': 710917276, 'L': 710917780, 'o': '56215.99000000', 'c': '56232.07000000', 'h': '56238.59000000', 'l': '56181.99000000', 'v': '13.80522200', 'n': 505, 'x': False, 'q': '775978.37383076', 'V': '7.19660600', 'Q': '404521.60814919', 'B': '0' } } """ if self.candle_closed_only and not msg['k']['x']: return c = Candle(self.id, self.exchange_symbol_to_std_symbol(msg['s']), msg['k']['t'] / 1000, msg['k']['T'] / 1000, msg['k']['i'], msg['k']['n'], Decimal(msg['k']['o']), Decimal(msg['k']['c']), Decimal(msg['k']['h']), Decimal(msg['k']['l']), Decimal(msg['k']['v']), msg['k']['x'], self.timestamp_normalize(msg['E']), raw=msg) await self.callback(CANDLES, c, timestamp) async def _account_update(self, msg: dict, timestamp: float): """ { "e": "outboundAccountPosition", //Event type "E": 1564034571105, //Event Time "u": 1564034571073, //Time of last account update "B": [ //Balances Array { "a": "ETH", //Asset "f": "10000.000000", //Free "l": "0.000000" //Locked } ] } """ for balance in msg['B']: b = Balance(self.id, balance['a'], Decimal(balance['f']), Decimal(balance['l']), raw=msg) await self.callback(BALANCES, b, timestamp) async def _order_update(self, msg: dict, timestamp: float): """ { "e": "executionReport", // Event type "E": 1499405658658, // Event time "s": "ETHBTC", // Symbol "c": "mUvoqJxFIILMdfAW5iGSOW", // Client order ID "S": "BUY", // Side "o": "LIMIT", // Order type "f": "GTC", // Time in force "q": "1.00000000", // Order quantity "p": "0.10264410", // Order price "P": "0.00000000", // Stop price "F": "0.00000000", // Iceberg quantity "g": -1, // OrderListId "C": "", // Original client order ID; This is the ID of the order being canceled "x": "NEW", // Current execution type "X": "NEW", // Current order status "r": "NONE", // Order reject reason; will be an error code. "i": 4293153, // Order ID "l": "0.00000000", // Last executed quantity "z": "0.00000000", // Cumulative filled quantity "L": "0.00000000", // Last executed price "n": "0", // Commission amount "N": null, // Commission asset "T": 1499405658657, // Transaction time "t": -1, // Trade ID "I": 8641984, // Ignore "w": true, // Is the order on the book? "m": false, // Is this trade the maker side? "M": false, // Ignore "O": 1499405658657, // Order creation time "Z": "0.00000000", // Cumulative quote asset transacted quantity "Y": "0.00000000", // Last quote asset transacted quantity (i.e. lastPrice * lastQty) "Q": "0.00000000" // Quote Order Qty } """ oi = OrderInfo(self.id, self.exchange_symbol_to_std_symbol(msg['s']), str(msg['i']), BUY if msg['S'].lower() == 'buy' else SELL, msg['x'], LIMIT if msg['o'].lower() == 'limit' else MARKET if msg['o'].lower() == 'market' else None, Decimal(msg['Z']) / Decimal(msg['z']) if not Decimal.is_zero(Decimal(msg['z'])) else None, Decimal(msg['q']), Decimal(msg['q']) - Decimal(msg['z']), self.timestamp_normalize(msg['E']), raw=msg) await self.callback(ORDER_INFO, oi, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) # Handle account updates from User Data Stream if self.requires_authentication: msg_type = msg['e'] if msg_type == 'outboundAccountPosition': await self._account_update(msg, timestamp) elif msg_type == 'executionReport': await self._order_update(msg, timestamp) return # Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>} # streamName is of format <symbol>@<channel> pair, _ = msg['stream'].split('@', 1) msg = msg['data'] pair = pair.upper() if 'e' in msg: if msg['e'] == 'depthUpdate': await self._book(msg, pair, timestamp) elif msg['e'] == 'aggTrade': await self._trade(msg, timestamp) elif msg['e'] == 'forceOrder': await self._liquidations(msg, timestamp) elif msg['e'] == 'markPriceUpdate': await self._funding(msg, timestamp) elif msg['e'] == 'kline': await self._candle(msg, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg) elif 'A' in msg: await self._ticker(msg, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): # Binance does not have a separate subscribe message, the # subscription information is included in the # connection endpoint if isinstance(conn, (HTTPPoll, HTTPConcurrentPoll)): self._open_interest_cache = {} else: self._reset() if self.requires_authentication: create_task(self._refresh_token())
class FTX(Feed, FTXRestMixin): id = FTX_id websocket_endpoints = [ WebsocketEndpoint('wss://ftx.com/ws/', options={'compression': None}) ] rest_endpoints = [ RestEndpoint('https://ftx.com', routes=Routes('/api/markets', open_interest='/api/futures/{}/stats', funding='/api/funding_rates?future={}', stats='/api/futures/{}/stats')) ] websocket_channels = { L2_BOOK: 'orderbook', TRADES: 'trades', TICKER: 'ticker', FUNDING: 'funding', OPEN_INTEREST: 'open_interest', LIQUIDATIONS: 'trades', ORDER_INFO: 'orders', FILLS: 'fills', } request_limit = 30 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for d in data['result']: if not d['enabled']: continue expiry = None stype = SPOT # FTX Futures contracts are stable coin settled, but # prices quoted are in USD, see https://help.ftx.com/hc/en-us/articles/360024780791-What-Are-Futures if "-MOVE-" in d['name']: stype = FUTURES base, expiry = d['name'].rsplit("-", maxsplit=1) quote = 'USD' if 'Q' in expiry: year, quarter = expiry.split("Q") year = year[2:] date = ["0325", "0624", "0924", "1231"] expiry = year + date[int(quarter) - 1] elif "-" in d['name']: base, expiry = d['name'].split("-") quote = 'USD' stype = FUTURES if expiry == 'PERP': expiry = None stype = PERPETUAL elif d['type'] == SPOT: base, quote = d['baseCurrency'], d['quoteCurrency'] else: # not enough info to construct a symbol - this is usually caused # by non crypto futures, i.e. TRUMP2024 or other contracts involving # betting on world events continue s = Symbol(base, quote, type=stype, expiry_date=expiry) ret[s.normalized] = d['name'] info['tick_size'][s.normalized] = d['priceIncrement'] info['quantity_step'][s.normalized] = d['sizeIncrement'] info['instrument_type'][s.normalized] = s.type return ret, info def __reset(self): self._l2_book = {} self._funding_cache = {} self._open_interest_cache = {} async def generate_token(self, conn: AsyncConnection): ts = int(time() * 1000) msg = { 'op': 'login', 'args': { 'key': self.key_id, 'sign': hmac.new(self.key_secret.encode(), f'{ts}websocket_login'.encode(), 'sha256').hexdigest(), 'time': ts, } } if self.subaccount: msg['args']['subaccount'] = self.subaccount await conn.write(json.dumps(msg)) async def authenticate(self, conn: AsyncConnection): if self.requires_authentication: await self.generate_token(conn) async def subscribe(self, conn: AsyncConnection): self.__reset() for chan in self.subscription: symbols = self.subscription[chan] if chan == FUNDING: asyncio.create_task(self._funding(symbols)) continue if chan == OPEN_INTEREST: asyncio.create_task(self._open_interest(symbols)) continue if self.is_authenticated_channel( self.exchange_channel_to_std(chan)): await conn.write( json.dumps({ "channel": chan, "op": "subscribe" })) continue for pair in symbols: await conn.write( json.dumps({ "channel": chan, "market": pair, "op": "subscribe" })) async def _open_interest(self, pairs: Iterable): """ { "success": true, "result": { "volume": 1000.23, "nextFundingRate": 0.00025, "nextFundingTime": "2019-03-29T03:00:00+00:00", "expirationPrice": 3992.1, "predictedExpirationPrice": 3993.6, "strikePrice": 8182.35, "openInterest": 21124.583 } } """ while True: for pair in pairs: # OI only for perp and futures, so check for / in pair name indicating spot if '/' in pair: continue data = await self.http_conn.read(self.rest_endpoints[0].route( 'open_interest', sandbox=self.sandbox).format(pair)) received = time() data = json.loads(data, parse_float=Decimal) if 'result' in data: oi = data['result']['openInterest'] if oi != self._open_interest_cache.get(pair, None): o = OpenInterest( self.id, self.exchange_symbol_to_std_symbol(pair), oi, None, raw=data) await self.callback(OPEN_INTEREST, o, received) self._open_interest_cache[pair] = oi await asyncio.sleep(1) await asyncio.sleep(60) async def _funding(self, pairs: Iterable): """ { "success": true, "result": [ { "future": "BTC-PERP", "rate": 0.0025, "time": "2019-06-02T08:00:00+00:00" } ] } """ while True: for pair in pairs: if '-PERP' not in pair: continue data = await self.http_conn.read(self.rest_endpoints[0].route( 'funding', sandbox=self.sandbox).format(pair)) data = json.loads(data, parse_float=Decimal) data2 = await self.http_conn.read(self.rest_endpoints[0].route( 'stats', sandbox=self.sandbox).format(pair)) data2 = json.loads(data2, parse_float=Decimal) received = time() data['predicted_rate'] = Decimal( data2['result']['nextFundingRate']) last_update = self._funding_cache.get(pair, None) update = str(data['result'][0]['rate']) + str( data['result'][0]['time']) + str(data['predicted_rate']) if last_update and last_update == update: continue else: self._funding_cache[pair] = update f = Funding(self.id, self.exchange_symbol_to_std_symbol( data['result'][0]['future']), None, data['result'][0]['rate'], self.timestamp_normalize( data2['result']['nextFundingTime']), self.timestamp_normalize( data['result'][0]['time']), predicted_rate=data['predicted_rate'], raw=[data, data2]) await self.callback(FUNDING, f, received) await asyncio.sleep(0.1) await asyncio.sleep(60) async def _trade(self, msg: dict, timestamp: float): """ example message: {"channel": "trades", "market": "BTC-PERP", "type": "update", "data": [{"id": null, "price": 10738.75, "size": 0.3616, "side": "buy", "liquidation": false, "time": "2019-08-03T12:20:19.170586+00:00"}]} """ for trade in msg['data']: t = Trade(self.id, self.exchange_symbol_to_std_symbol(msg['market']), BUY if trade['side'] == 'buy' else SELL, Decimal(trade['size']), Decimal(trade['price']), float(self.timestamp_normalize(trade['time'])), id=str(trade['id']), raw=trade) await self.callback(TRADES, t, timestamp) if bool(trade['liquidation']): liq = Liquidation( self.id, self.exchange_symbol_to_std_symbol(msg['market']), BUY if trade['side'] == 'buy' else SELL, Decimal(trade['size']), Decimal(trade['price']), str(trade['id']), FILLED, float(self.timestamp_normalize(trade['time'])), raw=trade) await self.callback(LIQUIDATIONS, liq, timestamp) async def _ticker(self, msg: dict, timestamp: float): """ example message: {"channel": "ticker", "market": "BTC/USD", "type": "update", "data": {"bid": 10717.5, "ask": 10719.0, "last": 10719.0, "time": 1564834587.1299787}} """ t = Ticker(self.id, self.exchange_symbol_to_std_symbol(msg['market']), Decimal(msg['data']['bid'] if msg['data']['bid'] else 0.0), Decimal(msg['data']['ask'] if msg['data']['ask'] else 0.0), float(msg['data']['time']), raw=msg) await self.callback(TICKER, t, timestamp) async def _book(self, msg: dict, timestamp: float): """ example messages: snapshot: {"channel": "orderbook", "market": "BTC/USD", "type": "partial", "data": {"time": 1564834586.3382702, "checksum": 427503966, "bids": [[10717.5, 4.092], ...], "asks": [[10720.5, 15.3458], ...], "action": "partial"}} update: {"channel": "orderbook", "market": "BTC/USD", "type": "update", "data": {"time": 1564834587.1299787, "checksum": 3115602423, "bids": [], "asks": [[10719.0, 14.7461]], "action": "update"}} """ check = msg['data']['checksum'] if msg['type'] == 'partial': # snapshot pair = self.exchange_symbol_to_std_symbol(msg['market']) self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, checksum_format='FTX') self._l2_book[pair].book.bids = { Decimal(price): Decimal(amount) for price, amount in msg['data']['bids'] } self._l2_book[pair].book.asks = { Decimal(price): Decimal(amount) for price, amount in msg['data']['asks'] } if self.checksum_validation and self._l2_book[pair].book.checksum( ) != check: raise BadChecksum await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=float(msg['data']['time']), raw=msg, checksum=check) else: # update delta = {BID: [], ASK: []} pair = self.exchange_symbol_to_std_symbol(msg['market']) for side in ('bids', 'asks'): s = BID if side == 'bids' else ASK for price, amount in msg['data'][side]: price = Decimal(price) amount = Decimal(amount) if amount == 0: delta[s].append((price, 0)) del self._l2_book[pair].book[s][price] else: delta[s].append((price, amount)) self._l2_book[pair].book[s][price] = amount if self.checksum_validation and self._l2_book[pair].book.checksum( ) != check: raise BadChecksum await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=float(msg['data']['time']), raw=msg, checksum=check, delta=delta) async def _fill(self, msg: dict, timestamp: float): """ example message: { "channel": "fills", "data": { "fee": 78.05799225, "feeRate": 0.0014, "future": "BTC-PERP", "id": 7828307, "liquidity": "taker", "market": "BTC-PERP", "orderId": 38065410, "tradeId": 19129310, "price": 3723.75, "side": "buy", "size": 14.973, "time": "2019-05-07T16:40:58.358438+00:00", "type": "order" }, "type": "update" } """ fill = msg['data'] f = Fill(self.id, self.exchange_symbol_to_std_symbol(fill['market']), BUY if fill['side'] == 'buy' else SELL, Decimal(fill['size']), Decimal(fill['price']), Decimal(fill['fee']), str(fill['tradeId']), str(fill['orderId']), None, TAKER if fill['liquidity'] == 'taker' else MAKER, fill['time'].timestamp(), account=self.subaccount, raw=msg) await self.callback(FILLS, f, timestamp) async def _order(self, msg: dict, timestamp: float): """ example message: { "channel": "orders", "data": { "id": 24852229, "clientId": null, "market": "XRP-PERP", "type": "limit", "side": "buy", "size": 42353.0, "price": 0.2977, "reduceOnly": false, "ioc": false, "postOnly": false, "status": "closed", "filledSize": 0.0, "remainingSize": 0.0, "avgFillPrice": 0.2978 }, "type": "update" } """ order = msg['data'] status = order['status'] if status == 'new': status = SUBMITTING elif status == 'open': status = OPEN elif status == 'closed': status = CLOSED oi = OrderInfo(self.id, self.exchange_symbol_to_std_symbol(order['market']), str(order['id']), BUY if order['side'].lower() == 'buy' else SELL, status, LIMIT if order['type'] == 'limit' else MARKET, Decimal(order['price']) if order['price'] else None, Decimal(order['filledSize']), Decimal(order['remainingSize']), None, account=self.subaccount, raw=msg) await self.callback(ORDER_INFO, oi, timestamp) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'type' in msg: if msg['type'] == 'subscribed': if 'market' in msg: LOG.info('%s: Subscribed to %s channel for %s', self.id, msg['channel'], msg['market']) else: LOG.info('%s: Subscribed to %s channel', self.id, msg['channel']) elif msg['type'] == 'error': LOG.error('%s: Received error message %s', self.id, msg) raise Exception('Error from %s: %s', self.id, msg) elif 'channel' in msg: if msg['channel'] == 'orderbook': await self._book(msg, timestamp) elif msg['channel'] == 'trades': await self._trade(msg, timestamp) elif msg['channel'] == 'ticker': await self._ticker(msg, timestamp) elif msg['channel'] == 'fills': await self._fill(msg, timestamp) elif msg['channel'] == 'orders': await self._order(msg, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) else: LOG.warning("%s: Invalid message type %s", self.id, msg) else: LOG.warning("%s: Invalid message type %s", self.id, msg)
class AscendEX(Feed): id = ASCENDEX websocket_endpoints = [WebsocketEndpoint('wss://ascendex.com/1/api/pro/v1/stream')] rest_endpoints = [RestEndpoint('https://ascendex.com', routes=Routes('/api/pro/v1/products'))] websocket_channels = { L2_BOOK: 'depth:', TRADES: 'trades:', } @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 @classmethod def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) for entry in data['data']: # Only "Normal" status symbols are tradeable if entry['status'] == 'Normal': s = Symbol(entry['baseAsset'], entry['quoteAsset']) ret[s.normalized] = entry['symbol'] info['tick_size'][s.normalized] = entry['tickSize'] info['instrument_type'][s.normalized] = s.type return ret, info def __reset(self): self._l2_book = {} self.seq_no = defaultdict(lambda: None) async def _trade(self, msg: dict, timestamp: float): """ { 'm': 'trades', 'symbol': 'BTC/USDT', 'data': [{ 'p': '23169.76', 'q': '0.00899', 'ts': 1608760026461, 'bm': False, 'seqnum': 72057614186183012 }] } """ for trade in msg['data']: t = Trade(self.id, self.exchange_symbol_to_std_symbol(msg['symbol']), SELL if trade['bm'] else BUY, Decimal(trade['q']), Decimal(trade['p']), self.timestamp_normalize(trade['ts']), raw=trade) await self.callback(TRADES, t, timestamp) async def _book(self, msg: dict, timestamp: float): sequence_number = msg['data']['seqnum'] pair = self.exchange_symbol_to_std_symbol(msg['symbol']) delta = {BID: [], ASK: []} if msg['m'] == 'depth-snapshot': self.seq_no[pair] = sequence_number self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) else: # ignore messages while we wait for the snapshot if self.seq_no[pair] is None: return if self.seq_no[pair] + 1 != sequence_number: raise MissingSequenceNumber self.seq_no[pair] = sequence_number for side in ('bids', 'asks'): for price, amount in msg['data'][side]: s = BID if side == 'bids' else ASK price = Decimal(price) size = Decimal(amount) if size == 0: delta[s].append((price, 0)) if price in self._l2_book[pair].book[s]: del self._l2_book[pair].book[s][price] else: delta[s].append((price, size)) self._l2_book[pair].book[s][price] = size await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(msg['data']['ts']), raw=msg, delta=delta if msg['m'] != 'depth-snapshot' else None, sequence_number=sequence_number) async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'm' in msg: if msg['m'] == 'depth' or msg['m'] == 'depth-snapshot': await self._book(msg, timestamp) elif msg['m'] == 'trades': await self._trade(msg, timestamp) elif msg['m'] == 'ping': await conn.write('{"op":"pong"}') elif msg['m'] == 'connected': return elif msg['m'] == 'sub': return else: LOG.warning("%s: Invalid message type %s", self.id, msg) else: LOG.warning("%s: Invalid message type %s", self.id, msg) async def subscribe(self, conn: AsyncConnection): self.__reset() l2_pairs = [] for channel in self.subscription: pairs = self.subscription[channel] if channel == "depth:": l2_pairs.extend(pairs) message = {'op': 'sub', 'ch': channel + ','.join(pairs)} await conn.write(json.dumps(message)) for pair in l2_pairs: message = {"op": "req", "action": "depth-snapshot", "args": {"symbol": pair}} await conn.write(json.dumps(message))
class Deribit(Feed, DeribitRestMixin): id = DERIBIT websocket_endpoints = [ WebsocketEndpoint('wss://www.deribit.com/ws/api/v2', sandbox='wss://test.deribit.com/ws/api/v2') ] rest_endpoints = [ RestEndpoint('https://www.deribit.com', sandbox='https://test.deribit.com', routes=Routes([ '/api/v2/public/get_instruments?currency=BTC', '/api/v2/public/get_instruments?currency=ETH', '/api/v2/public/get_instruments?currency=USDC', '/api/v2/public/get_instruments?currency=SOL' ])) ] websocket_channels = { L1_BOOK: 'quote', L2_BOOK: 'book', TRADES: 'trades', TICKER: 'ticker', FUNDING: 'ticker', OPEN_INTEREST: 'ticker', LIQUIDATIONS: 'trades', ORDER_INFO: 'user.orders', FILLS: 'user.trades', BALANCES: 'user.portfolio', } request_limit = 20 @classmethod def timestamp_normalize(cls, ts: float) -> float: return ts / 1000.0 @classmethod def is_authenticated_channel(cls, channel: str) -> bool: return channel in (ORDER_INFO, FILLS, BALANCES, L2_BOOK, TRADES, TICKER) @classmethod def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]: ret = {} info = defaultdict(dict) currencies = [] for entry in data: for e in entry['result']: base = e['base_currency'] if base not in currencies: currencies.append(base) quote = e['quote_currency'] stype = e['kind'] if e[ 'settlement_period'] != 'perpetual' else PERPETUAL otype = e.get('option_type') strike_price = e.get('strike') strike_price = int(strike_price) if strike_price else None expiry = e['expiration_timestamp'] / 1000 s = Symbol(base, quote, type=FUTURES if stype == 'future' else stype, option_type=otype, strike_price=strike_price, expiry_date=expiry) ret[s.normalized] = e['instrument_name'] info['tick_size'][s.normalized] = e['tick_size'] info['instrument_type'][s.normalized] = stype for currency in currencies: s = Symbol(currency, currency, type=CURRENCY) ret[s.normalized] = currency info['instrument_type'][s.normalized] = CURRENCY return ret, info def __reset(self): self._open_interest_cache = {} self._l2_book = {} self.seq_no = {} async def _trade(self, msg: dict, timestamp: float): """ { "params": { "data": [ { "trade_seq": 933, "trade_id": "9178", "timestamp": 1550736299753, "tick_direction": 3, "price": 3948.69, "instrument_name": "BTC-PERPETUAL", "index_price": 3930.73, "direction": "sell", "amount": 10 } ], "channel": "trades.BTC-PERPETUAL.raw" }, "method": "subscription", "jsonrpc": "2.0" } """ for trade in msg["params"]["data"]: t = Trade(self.id, self.exchange_symbol_to_std_symbol( trade["instrument_name"]), BUY if trade['direction'] == 'buy' else SELL, Decimal(trade['amount']), Decimal(trade['price']), self.timestamp_normalize(trade['timestamp']), id=trade['trade_id'], raw=trade) await self.callback(TRADES, t, timestamp) if 'liquidation' in trade: liq = Liquidation(self.id, self.exchange_symbol_to_std_symbol( trade["instrument_name"]), BUY if trade['direction'] == 'buy' else SELL, Decimal(trade['amount']), Decimal(trade['price']), trade['trade_id'], FILLED, self.timestamp_normalize(trade['timestamp']), raw=trade) await self.callback(LIQUIDATIONS, liq, timestamp) async def _ticker(self, msg: dict, timestamp: float): ''' { "params" : { "data" : { "timestamp" : 1550652954406, "stats" : { "volume" : null, "low" : null, "high" : null }, "state" : "open", "settlement_price" : 3960.14, "open_interest" : 0.12759952124659626, "min_price" : 3943.21, "max_price" : 3982.84, "mark_price" : 3940.06, "last_price" : 3906, "instrument_name" : "BTC-PERPETUAL", "index_price" : 3918.51, "funding_8h" : 0.01520525, "current_funding" : 0.00499954, "best_bid_price" : 3914.97, "best_bid_amount" : 40, "best_ask_price" : 3996.61, "best_ask_amount" : 50 }, "channel" : "ticker.BTC-PERPETUAL.raw" }, "method" : "subscription", "jsonrpc" : "2.0"} ''' pair = self.exchange_symbol_to_std_symbol( msg['params']['data']['instrument_name']) ts = self.timestamp_normalize(msg['params']['data']['timestamp']) t = Ticker(self.id, pair, Decimal(msg["params"]["data"]['best_bid_price']), Decimal(msg["params"]["data"]['best_ask_price']), ts, raw=msg) await self.callback(TICKER, t, timestamp) if "current_funding" in msg["params"]["data"] and "funding_8h" in msg[ "params"]["data"]: f = Funding(self.id, pair, Decimal(msg['params']['data']['mark_price']), Decimal(msg["params"]["data"]["current_funding"]), None, ts, raw=msg) await self.callback(FUNDING, f, timestamp) oi = msg['params']['data']['open_interest'] if pair in self._open_interest_cache and oi == self._open_interest_cache[ pair]: return self._open_interest_cache[pair] = oi o = OpenInterest(self.id, pair, Decimal(oi), ts, raw=msg) await self.callback(OPEN_INTEREST, o, timestamp) async def _quote(self, quote: dict, timestamp: float): book = L1Book(self.id, self.exchange_symbol_to_std_symbol( quote['instrument_name']), Decimal(quote['best_bid_price']), Decimal(quote['best_bid_amount']), Decimal(quote['best_ask_price']), Decimal(quote['best_ask_amount']), self.timestamp_normalize(quote['timestamp']), raw=quote) await self.callback(L1_BOOK, book, timestamp) async def subscribe(self, conn: AsyncConnection): self.__reset() pub_channels = [] pri_channels = [] for chan in self.subscription: if self.is_authenticated_channel( self.exchange_channel_to_std(chan)): for pair in self.subscription[chan]: if self.exchange_channel_to_std(chan) == BALANCES: pri_channels.append(f"{chan}.{pair}") else: pri_channels.append(f"{chan}.{pair}.raw") else: for pair in self.subscription[chan]: if self.exchange_channel_to_std(chan) == L1_BOOK: pub_channels.append(f"{chan}.{pair}") else: pub_channels.append(f"{chan}.{pair}.raw") if pub_channels: msg = { "jsonrpc": "2.0", "id": "101", "method": "public/subscribe", "params": { "channels": pub_channels } } LOG.debug( f'{conn.uuid}: Subscribing to public channels with message {msg}' ) await conn.write(json.dumps(msg)) if pri_channels: msg = { "jsonrpc": "2.0", "id": "102", "method": "private/subscribe", "params": { "scope": f"session:{conn.uuid}", "channels": pri_channels } } LOG.debug( f'{conn.uuid}: Subscribing to private channels with message {msg}' ) await conn.write(json.dumps(msg)) async def _book_snapshot(self, msg: dict, timestamp: float): """ { 'jsonrpc': '2.0', 'method': 'subscription', 'params': { 'channel': 'book.BTC-PERPETUAL.raw', 'data': { 'timestamp': 1598232105378, 'instrument_name': 'BTC-PERPETUAL', 'change_id': 21486665526, ' 'bids': [['new', Decimal('11618.5'), Decimal('279310.0')], ..... ] 'asks': [[ ....... ]] } } } """ ts = msg["params"]["data"]["timestamp"] pair = self.exchange_symbol_to_std_symbol( msg["params"]["data"]["instrument_name"]) self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth) self._l2_book[pair].book.bids = { Decimal(price): Decimal(amount) for _, price, amount in msg["params"]["data"]["bids"] } self._l2_book[pair].book.asks = { Decimal(price): Decimal(amount) for _, price, amount in msg["params"]["data"]["asks"] } self.seq_no[pair] = msg["params"]["data"]["change_id"] await self.book_callback( L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(ts), sequence_number=msg["params"]["data"]["change_id"], raw=msg) async def _book_update(self, msg: dict, timestamp: float): ts = msg["params"]["data"]["timestamp"] pair = self.exchange_symbol_to_std_symbol( msg["params"]["data"]["instrument_name"]) if msg['params']['data']['prev_change_id'] != self.seq_no[pair]: LOG.warning("%s: Missing sequence number detected for %s", self.id, pair) LOG.warning("%s: Requesting book snapshot", self.id) raise MissingSequenceNumber self.seq_no[pair] = msg['params']['data']['change_id'] delta = {BID: [], ASK: []} for action, price, amount in msg["params"]["data"]["bids"]: if action != "delete": self._l2_book[pair].book.bids[price] = Decimal(amount) delta[BID].append((Decimal(price), Decimal(amount))) else: del self._l2_book[pair].book.bids[price] delta[BID].append((Decimal(price), Decimal(amount))) for action, price, amount in msg["params"]["data"]["asks"]: if action != "delete": self._l2_book[pair].book.asks[price] = amount delta[ASK].append((Decimal(price), Decimal(amount))) else: del self._l2_book[pair].book.asks[price] delta[ASK].append((Decimal(price), Decimal(amount))) await self.book_callback( L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(ts), raw=msg, delta=delta, sequence_number=msg['params']['data']['change_id']) async def message_handler(self, msg: str, conn, timestamp: float): msg_dict = json.loads(msg, parse_float=Decimal) if 'error' in msg_dict.keys(): LOG.error("%s: Received Error message: %s, Error code: %s", conn.uuid, msg_dict['error']['message'], msg_dict['error']['code']) elif "result" in msg_dict: result = msg_dict["result"] if 'id' in msg_dict: id = str(msg_dict["id"]) if id == "0": LOG.debug("%s: Connected", conn.uuid) elif id == '101': LOG.info("%s: Subscribed to public channels", conn.uuid) LOG.debug("%s: %s", conn.uuid, result) elif id == '102': LOG.info("%s: Subscribed to authenticated channels", conn.uuid) LOG.debug("%s: %s", conn.uuid, result) elif id.startswith('auth') and "access_token" in result: ''' Access token is another way to be authenticated while sending messages to Deribit. In this implementation 'scope session' method is used instead of 'acces token' method. ''' LOG.debug(f"{conn.uuid}: Access token received") else: LOG.warning("%s: Unknown id in message %s", conn.uuid, msg_dict) else: LOG.warning("%s: Unknown 'result' message received: %s", conn.uuid, msg_dict) elif 'params' in msg_dict: params = msg_dict['params'] if 'channel' in params: channel = params['channel'] if "ticker" == channel.split(".")[0]: await self._ticker(msg_dict, timestamp) elif "trades" == channel.split(".")[0]: await self._trade(msg_dict, timestamp) elif "book" == channel.split(".")[0]: # checking if we got full book or its update # if it's update there is 'prev_change_id' field if "prev_change_id" not in msg_dict["params"]["data"].keys( ): await self._book_snapshot(msg_dict, timestamp) elif "prev_change_id" in msg_dict["params"]["data"].keys(): await self._book_update(msg_dict, timestamp) elif "quote" == channel.split(".")[0]: await self._quote(params['data'], timestamp) elif channel.startswith("user"): await self._user_channels(conn, msg_dict, timestamp, channel.split(".")[1]) else: LOG.warning("%s: Unknown channel %s", conn.uuid, msg_dict) else: LOG.warning("%s: Unknown 'params' message received: %s", conn.uuid, msg_dict) else: LOG.warning("%s: Unknown message in msg_dict: %s", conn.uuid, msg_dict) async def authenticate(self, conn: AsyncConnection): if self.requires_authentication: auth = self._auth(self.key_id, self.key_secret, conn.uuid) LOG.debug(f"{conn.uuid}: Authenticating with message: {auth}") await conn.write(json.dumps(auth)) def _auth(self, key_id, key_secret, session_id: str) -> str: # https://docs.deribit.com/?python#authentication timestamp = round(datetime.now().timestamp() * 1000) nonce = f'xyz{str(timestamp)[-5:]}' signature = hmac.new(bytes(key_secret, "latin-1"), bytes('{}\n{}\n{}'.format(timestamp, nonce, ""), "latin-1"), digestmod=hashlib.sha256).hexdigest().lower() auth = { "jsonrpc": "2.0", "id": f"auth_{session_id}", "method": "public/auth", "params": { "grant_type": "client_signature", "client_id": key_id, "timestamp": timestamp, "signature": signature, "nonce": nonce, "data": "", "scope": f"session:{session_id}" } } return auth async def _user_channels(self, conn: AsyncConnection, msg: dict, timestamp: float, subchan: str): order_status = { "open": OPEN, "filled": FILLED, "rejected": FAILED, "cancelled": CANCELLED, "untriggered": OPEN } order_types = { "limit": LIMIT, "market": MARKET, "stop_limit": STOP_LIMIT, "stop_market": STOP_MARKET } if 'data' in msg['params']: data = msg['params']['data'] if subchan == 'portfolio': ''' { "params" : { "data" : { "total_pl" : 0.00000425, "session_upl" : 0.00000425, "session_rpl" : -2e-8, "projected_maintenance_margin" : 0.00009141, "projected_initial_margin" : 0.00012542, "projected_delta_total" : 0.0043, "portfolio_margining_enabled" : false, "options_vega" : 0, "options_value" : 0, "options_theta" : 0, "options_session_upl" : 0, "options_session_rpl" : 0, "options_pl" : 0, "options_gamma" : 0, "options_delta" : 0, "margin_balance" : 0.2340038, "maintenance_margin" : 0.00009141, "initial_margin" : 0.00012542, "futures_session_upl" : 0.00000425, "futures_session_rpl" : -2e-8, "futures_pl" : 0.00000425, "estimated_liquidation_ratio" : 0.01822795, "equity" : 0.2340038, "delta_total" : 0.0043, "currency" : "BTC", "balance" : 0.23399957, "available_withdrawal_funds" : 0.23387415, "available_funds" : 0.23387838 }, "channel" : "user.portfolio.btc" }, "method" : "subscription", "jsonrpc" : "2.0" } ''' b = Balance(self.id, data['currency'], Decimal(data['balance']), Decimal(data['balance']) - Decimal(data['available_withdrawal_funds']), raw=data) await self.callback(BALANCES, b, timestamp) elif subchan == 'orders': ''' { "params" : { "data" : { "time_in_force" : "good_til_cancelled", "replaced" : false, "reduce_only" : false, "profit_loss" : 0, "price" : 10502.52, "post_only" : false, "original_order_type" : "market", "order_type" : "limit", "order_state" : "open", "order_id" : "5", "max_show" : 200, "last_update_timestamp" : 1581507423789, "label" : "", "is_liquidation" : false, "instrument_name" : "BTC-PERPETUAL", "filled_amount" : 0, "direction" : "buy", "creation_timestamp" : 1581507423789, "commission" : 0, "average_price" : 0, "api" : false, "amount" : 200 }, "channel" : "user.orders.BTC-PERPETUAL.raw" }, "method" : "subscription", "jsonrpc" : "2.0" } ''' oi = OrderInfo( self.id, self.exchange_symbol_to_std_symbol( data['instrument_name']), data["order_id"], BUY if msg["side"] == 'Buy' else SELL, order_status[data["order_state"]], order_types[data['order_type']], Decimal(data['price']), Decimal(data['filled_amount']), Decimal(data['amount']) - Decimal(data['cumQuantity']), self.timestamp_normalize(data["last_update_timestamp"]), raw=data) await self.callback(ORDER_INFO, oi, timestamp) elif subchan == 'trades': ''' { "params" : { "data" : [ { "trade_seq" : 30289432, "trade_id" : "48079254", "timestamp" : 1590484156350, "tick_direction" : 0, "state" : "filled", "self_trade" : false, "reduce_only" : false, "price" : 8954, "post_only" : false, "order_type" : "market", "order_id" : "4008965646", "matching_id" : null, "mark_price" : 8952.86, "liquidity" : "T", "instrument_name" : "BTC-PERPETUAL", "index_price" : 8956.73, "fee_currency" : "BTC", "fee" : 0.00000168, "direction" : "sell", "amount" : 20 }] } } ''' for entry in data: symbol = self.exchange_symbol_to_std_symbol( entry['instrument_name']) f = Fill(self.id, symbol, SELL if entry['direction'] == 'sell' else BUY, Decimal(entry['amount']), Decimal(entry['price']), Decimal(entry['fee']), entry['trade_id'], entry['order_id'], entry['order_type'], TAKER if entry['liquidity'] == 'T' else MAKER, self.timestamp_normalize(entry['timestamp']), raw=entry) await self.callback(FILLS, f, timestamp) else: LOG.warning("%s: Unknown channel 'user.%s'", conn.uuid, subchan) else: LOG.warning("%s: Unknown message %s'", conn.uuid, msg)