def _address(self) -> Union[str, Dict]: """ Binance has a 200 pair/stream limit per connection, so we need to break the address down into multiple connections if necessary. Because the key is currently not used for the address dict, we can just set it to the last used stream, since this will be unique. The generic connect method supplied by Feed will take care of creating the correct connection objects from the addresses. """ if self.requires_authentication: listen_key = self.auth.generate_token() address = self.ws_endpoint + '/ws/' + listen_key else: address = self.ws_endpoint + '/stream?streams=' subs = [] is_any_private = any(is_authenticated_channel(chan) for chan in self.subscription) is_any_public = any(not is_authenticated_channel(chan) for chan in self.subscription) if is_any_private and is_any_public: raise ValueError("Private and public channels should be subscribed to in separate feeds") for chan in self.subscription: normalized_chan = normalize_channel(self.id, chan) if normalized_chan == OPEN_INTEREST: continue if is_authenticated_channel(normalized_chan): continue stream = chan if normalized_chan == CANDLES: stream = f"{chan}{self.candle_interval}" for pair in self.subscription[chan]: # for everything but premium index the symbols need to be lowercase. if pair.startswith("p"): if normalized_chan != CANDLES: raise ValueError("Premium Index Symbols only allowed on Candle data feed") else: pair = pair.lower() sub = f"{pair}@{stream}" if normalized_chan == FUTURES_INDEX: pair = pair.split('_')[0] sub = f"{pair}@{stream}@1s" subs.append(sub) if len(subs) < 200: return address + '/'.join(subs) else: def split_list(_list: list, n: int): for i in range(0, len(_list), n): yield _list[i:i + n] return {chunk[0]: address + '/'.join(chunk) for chunk in split_list(subs, 200)}
async def subscribe(self, conn: AsyncConnection): self.__reset() for chan in self.subscription: symbols = self.subscription[chan] if chan == FUNDING: asyncio.create_task( self._funding(symbols)) # TODO: use HTTPAsyncConn continue if chan == OPEN_INTEREST: asyncio.create_task( self._open_interest(symbols)) # TODO: use HTTPAsyncConn continue if is_authenticated_channel(normalize_channel(self.id, chan)): await conn.write( json.dumps({ "channel": chan, "op": "subscribe" })) continue for pair in symbols: await conn.write( json.dumps({ "channel": chan, "market": pair, "op": "subscribe" }))
def connect( self ) -> List[Tuple[AsyncConnection, Callable[[None], None], Callable[ [str, float], None]]]: authenticated = [] public = [] ret = [] for channel in self.subscription or self.channels: if is_authenticated_channel(channel): authenticated.extend( self.subscription.get(channel) or self.symbols) else: public.extend(self.subscription.get(channel) or self.symbols) if authenticated: header = generate_token(self.key_id, self.key_secret, "/v1/order/events", self.config.gemini.account_name) symbols = '&'.join([ f"symbolFilter={s.lower()}" for s in authenticated ]) # needs to match REST format (lower case) ret.append( self._connect_builder(f"{self.address['auth']}?{symbols}", None, header=header, sub=self._empty_subscribe, handler=self.message_handler_orders)) if public: ret.append( self._connect_builder(self.address['public'], list(set(public)))) return ret
def get_channel_symbol_combinations(self): for chan in self.subscription: if not is_authenticated_channel(chan): if chan == LIQUIDATIONS: continue for symbol in self.subscription[chan]: instrument_type = self.instrument_type(symbol) if instrument_type != 'swap' and 'funding' in chan: continue # No funding for spot, futures and options yield f"{chan.format(instrument_type)}:{symbol}"
def connect(self) -> List[Tuple[AsyncConnection, Callable[[None], None], Callable[[str, float], None]]]: ret = [] for channel in self.subscription: if is_authenticated_channel(channel): for s in self.subscription[channel]: ret.append((WSAsyncConn(self.address, self.id, **self.ws_defaults), partial(self.user_order_subscribe, symbol=s), self.message_handler, self.authenticate)) else: ret.append((WSAsyncConn(self.address, self.id, **self.ws_defaults), self.subscribe, self.message_handler, self.authenticate)) return ret
async def update_subscription(self, subscription=None): normalized_subscription = defaultdict(set) for channel in subscription: chan = feed_to_exchange(self.id, channel) if is_authenticated_channel(channel): if not self.key_id or not self.key_secret: raise ValueError("Authenticated channel subscribed to, but no auth keys provided") normalized_subscription[chan].update([self.std_symbol_to_exchange_symbol(symbol, self.id) for symbol in subscription[channel]]) channels_to_subscribe = [] channels_to_unsubscribe = [] for chan in normalized_subscription: for pair in (set(normalized_subscription[chan]) - set(self.subscription[chan])): channels_to_subscribe.append(Deribit.build_channel_name(chan, pair)) for pair in (set(self.subscription[chan]) - set(normalized_subscription[chan])): channels_to_unsubscribe.append(Deribit.build_channel_name(chan, pair)) self.subscription = normalized_subscription LOG.info(f"Updating subscription. Channels to subscribe: {channels_to_subscribe}. Channels to unsubscribe: {channels_to_unsubscribe}") await self.subscribe_inner(self.connection, channels_to_subscribe) await self.unsubscribe_inner(self.connection, channels_to_unsubscribe)
def __init__(self, address: Union[dict, str], timeout=120, timeout_interval=30, retries=10, symbols=None, channels=None, subscription=None, config: Union[Config, dict, str] = None, callbacks=None, max_depth=None, book_interval=1000, snapshot_interval=False, checksum_validation=False, cross_check=False, origin=None, exceptions=None, log_message_on_error=False, sandbox=False): """ address: str, or dict address to be used to create the connection. The address protocol (wss or https) will be used to determine the connection type. Use a "str" to pass one single address, or a dict of option/address timeout: int Time, in seconds, between message to wait before a feed is considered dead and will be restarted. Set to -1 for infinite. timeout_interval: int Time, in seconds, between timeout checks. retries: int Number of times to retry a failed connection. Set to -1 for infinite max_depth: int Maximum number of levels per side to return in book updates book_interval: int Number of updates between snapshots. Only applicable when book deltas are enabled. Book deltas are enabled by subscribing to the book delta callback. snapshot_interval: bool/int Number of updates between snapshots. Only applicable when book delta is not enabled. Updates between snapshots are not delivered to the client checksum_validation: bool Toggle checksum validation, when supported by an exchange. cross_check: bool Toggle a check for a crossed book. Should not be needed on exchanges that support checksums or provide message sequence numbers. origin: str Passed into websocket connect. Sets the origin header. exceptions: list of exceptions These exceptions will not be handled internally and will be passed to the asyncio exception handler. To handle them feedhandler will need to be supplied with a custom exception handler. See the `run` method on FeedHandler, specifically the `exception_handler` keyword argument. log_message_on_error: bool If an exception is encountered in the connection handler, log the raw message sandbox: bool enable sandbox mode for exchanges that support this """ if isinstance(config, Config): LOG.info( '%s: reuse object Config containing the following main keys: %s', self.id, ", ".join(config.config.keys())) self.config = config else: LOG.info('%s: create Config from type: %r', self.id, type(config)) self.config = Config(config) self.sandbox = sandbox self.log_on_error = log_message_on_error self.retries = retries self.exceptions = exceptions self.connection_handlers = [] self.timeout = timeout self.timeout_interval = timeout_interval self.subscription = defaultdict(set) self.address = address self.book_update_interval = book_interval self.snapshot_interval = snapshot_interval self.cross_check = cross_check self.updates = defaultdict(int) self.do_deltas = False self.normalized_symbols = [] self.max_depth = max_depth self.previous_book = defaultdict(dict) self.origin = origin self.checksum_validation = checksum_validation self.ws_defaults = { 'ping_interval': 10, 'ping_timeout': None, 'max_size': 2**23, 'max_queue': None, 'origin': self.origin } self.key_id = os.environ.get(f'CF_{self.id}_KEY_ID') or self.config[ self.id.lower()].key_id self.key_secret = os.environ.get( f'CF_{self.id}_KEY_SECRET') or self.config[ self.id.lower()].key_secret self._feed_config = defaultdict(list) self.http_conn = HTTPAsyncConn(self.id) symbols_cache = Symbols if not symbols_cache.populated(self.id): self.symbol_mapping() self.normalized_symbol_mapping, self.exchange_info = symbols_cache.get( self.id) self.exchange_symbol_mapping = { value: key for key, value in self.normalized_symbol_mapping.items() } if subscription is not None and (symbols is not None or channels is not None): raise ValueError( "Use subscription, or channels and symbols, not both") if subscription is not None: self.channels = list(subscription.keys()) self.symbols = list(subscription[self.channels[0]]) for channel in subscription: chan = feed_to_exchange(self.id, channel) if is_authenticated_channel(channel): if not self.key_id or not self.key_secret: raise ValueError( "Authenticated channel subscribed to, but no auth keys provided" ) self.normalized_symbols.extend(subscription[channel]) self.subscription[chan].update([ self.std_symbol_to_exchange_symbol(symbol) for symbol in subscription[channel] ]) self._feed_config[channel].extend(self.normalized_symbols) if symbols and channels: if any(is_authenticated_channel(chan) for chan in channels): if not self.key_id or not self.key_secret: raise ValueError( "Authenticated channel subscribed to, but no auth keys provided" ) self.channels = channels self.symbols = symbols # if we dont have a subscription dict, we'll use symbols+channels and build one [ self._feed_config[channel].extend(symbols) for channel in channels ] self.normalized_symbols = symbols symbols = [ self.std_symbol_to_exchange_symbol(symbol) for symbol in symbols ] channels = list( set([feed_to_exchange(self.id, chan) for chan in channels])) self.subscription = {chan: symbols for chan in channels} self._feed_config = dict(self._feed_config) self.l3_book = {} self.l2_book = {} self.callbacks = { FUNDING: Callback(None), FUTURES_INDEX: Callback(None), L2_BOOK: Callback(None), L3_BOOK: Callback(None), LIQUIDATIONS: Callback(None), OPEN_INTEREST: Callback(None), MARKET_INFO: Callback(None), TICKER: Callback(None), TRADES: Callback(None), CANDLES: Callback(None), ORDER_INFO: Callback(None) } if callbacks: for cb_type, cb_func in callbacks.items(): self.callbacks[cb_type] = cb_func if cb_type == BOOK_DELTA: self.do_deltas = True for key, callback in self.callbacks.items(): if not isinstance(callback, list): self.callbacks[key] = [callback]
def __init__(self, address: Union[dict, str], sandbox=False, symbols=None, channels=None, subscription=None, config: Union[Config, dict, str] = None, callbacks=None, max_depth=None, book_interval=1000, snapshot_interval=False, checksum_validation=False, cross_check=False, origin=None): """ address: str, or dict address to be used to create the connection. The address protocol (wss or https) will be used to determine the connection type. Use a "str" to pass one single address, or a dict of option/address sandbox: bool For authenticated channels, run against the sandbox websocket (when True) max_depth: int Maximum number of levels per side to return in book updates book_interval: int Number of updates between snapshots. Only applicable when book deltas are enabled. Book deltas are enabled by subscribing to the book delta callback. snapshot_interval: bool/int Number of updates between snapshots. Only applicable when book delta is not enabled. Updates between snapshots are not delivered to the client checksum_validation: bool Toggle checksum validation, when supported by an exchange. cross_check: bool Toggle a check for a crossed book. Should not be needed on exchanges that support checksums or provide message sequence numbers. origin: str Passed into websocket connect. Sets the origin header. """ if isinstance(config, Config): LOG.info( '%s: reuse object Config containing the following main keys: %s', self.id, ", ".join(config.config.keys())) self.config = config else: LOG.info('%s: create Config from type: %r', self.id, type(config)) self.config = Config(config) self.subscription = defaultdict(set) self.address = address self.book_update_interval = book_interval self.snapshot_interval = snapshot_interval self.cross_check = cross_check self.updates = defaultdict(int) self.do_deltas = False self.symbols = [] self.normalized_symbols = [] self.channels = [] self.max_depth = max_depth self.previous_book = defaultdict(dict) self.origin = origin self.checksum_validation = checksum_validation self.ws_defaults = { 'ping_interval': 10, 'ping_timeout': None, 'max_size': 2**23, 'max_queue': None, 'origin': self.origin } self.key_id = os.environ.get(f'CF_{self.id}_KEY_ID') or self.config[ self.id.lower()].key_id self.key_secret = os.environ.get( f'CF_{self.id}_KEY_SECRET') or self.config[ self.id.lower()].key_secret self._feed_config = defaultdict(list) load_exchange_symbol_mapping(self.id, key_id=self.key_id) if subscription is not None and (symbols is not None or channels is not None): raise ValueError( "Use subscription, or channels and symbols, not both") if subscription is not None: for channel in subscription: chan = feed_to_exchange(self.id, channel) if is_authenticated_channel(channel): if not self.key_id or not self.key_secret: raise ValueError( "Authenticated channel subscribed to, but no auth keys provided" ) self.normalized_symbols.extend(subscription[channel]) self.subscription[chan].update([ symbol_std_to_exchange(symbol, self.id) for symbol in subscription[channel] ]) self._feed_config[channel].extend(self.normalized_symbols) if symbols: self.normalized_symbols = symbols self.symbols = [ symbol_std_to_exchange(symbol, self.id) for symbol in symbols ] if channels: self.channels = list( set([feed_to_exchange(self.id, chan) for chan in channels])) [ self._feed_config[channel].extend(self.normalized_symbols) for channel in channels ] if any(is_authenticated_channel(chan) for chan in channels): if not self.key_id or not self.key_secret: raise ValueError( "Authenticated channel subscribed to, but no auth keys provided" ) self._feed_config = dict(self._feed_config) self.l3_book = {} self.l2_book = {} self.callbacks = { FUNDING: Callback(None), FUTURES_INDEX: Callback(None), L2_BOOK: Callback(None), L3_BOOK: Callback(None), LIQUIDATIONS: Callback(None), OPEN_INTEREST: Callback(None), MARKET_INFO: Callback(None), TICKER: Callback(None), TRADES: Callback(None), TRANSACTIONS: Callback(None), VOLUME: Callback(None), CANDLES: Callback(None), ORDER_INFO: Callback(None) } if callbacks: for cb_type, cb_func in callbacks.items(): self.callbacks[cb_type] = cb_func if cb_type == BOOK_DELTA: self.do_deltas = True for key, callback in self.callbacks.items(): if not isinstance(callback, list): self.callbacks[key] = [callback]