def _address(self) -> Union[str, Dict]:
"""
Binance has a 200 pair/stream limit per connection, so we need to break the address
down into multiple connections if necessary. Because the key is currently not used
for the address dict, we can just set it to the last used stream, since this will be
unique.
The generic connect method supplied by Feed will take care of creating the
correct connection objects from the addresses.
"""
if self.requires_authentication:
listen_key = self.auth.generate_token()
address = self.ws_endpoint + '/ws/' + listen_key
else:
address = self.ws_endpoint + '/stream?streams='
subs = []
is_any_private = any(is_authenticated_channel(chan) for chan in self.subscription)
is_any_public = any(not is_authenticated_channel(chan) for chan in self.subscription)
if is_any_private and is_any_public:
raise ValueError("Private and public channels should be subscribed to in separate feeds")
for chan in self.subscription:
normalized_chan = normalize_channel(self.id, chan)
if normalized_chan == OPEN_INTEREST:
continue
if is_authenticated_channel(normalized_chan):
continue
stream = chan
if normalized_chan == CANDLES:
stream = f"{chan}{self.candle_interval}"
for pair in self.subscription[chan]:
# for everything but premium index the symbols need to be lowercase.
if pair.startswith("p"):
if normalized_chan != CANDLES:
raise ValueError("Premium Index Symbols only allowed on Candle data feed")
else:
pair = pair.lower()
sub = f"{pair}@{stream}"
if normalized_chan == FUTURES_INDEX:
pair = pair.split('_')[0]
sub = f"{pair}@{stream}@1s"
subs.append(sub)
if len(subs) < 200:
return address + '/'.join(subs)
else:
def split_list(_list: list, n: int):
for i in range(0, len(_list), n):
yield _list[i:i + n]
return {chunk[0]: address + '/'.join(chunk) for chunk in split_list(subs, 200)}
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
symbols = self.subscription[chan]
if chan == FUNDING:
asyncio.create_task(
self._funding(symbols)) # TODO: use HTTPAsyncConn
continue
if chan == OPEN_INTEREST:
asyncio.create_task(
self._open_interest(symbols)) # TODO: use HTTPAsyncConn
continue
if is_authenticated_channel(normalize_channel(self.id, chan)):
await conn.write(
json.dumps({
"channel": chan,
"op": "subscribe"
}))
continue
for pair in symbols:
await conn.write(
json.dumps({
"channel": chan,
"market": pair,
"op": "subscribe"
}))
def connect(
self
) -> List[Tuple[AsyncConnection, Callable[[None], None], Callable[
[str, float], None]]]:
authenticated = []
public = []
ret = []
for channel in self.subscription or self.channels:
if is_authenticated_channel(channel):
authenticated.extend(
self.subscription.get(channel) or self.symbols)
else:
public.extend(self.subscription.get(channel) or self.symbols)
if authenticated:
header = generate_token(self.key_id, self.key_secret,
"/v1/order/events",
self.config.gemini.account_name)
symbols = '&'.join([
f"symbolFilter={s.lower()}" for s in authenticated
]) # needs to match REST format (lower case)
ret.append(
self._connect_builder(f"{self.address['auth']}?{symbols}",
None,
header=header,
sub=self._empty_subscribe,
handler=self.message_handler_orders))
if public:
ret.append(
self._connect_builder(self.address['public'],
list(set(public))))
return ret
def get_channel_symbol_combinations(self):
for chan in self.subscription:
if not is_authenticated_channel(chan):
if chan == LIQUIDATIONS:
continue
for symbol in self.subscription[chan]:
instrument_type = self.instrument_type(symbol)
if instrument_type != 'swap' and 'funding' in chan:
continue # No funding for spot, futures and options
yield f"{chan.format(instrument_type)}:{symbol}"
def connect(self) -> List[Tuple[AsyncConnection, Callable[[None], None], Callable[[str, float], None]]]:
ret = []
for channel in self.subscription:
if is_authenticated_channel(channel):
for s in self.subscription[channel]:
ret.append((WSAsyncConn(self.address, self.id, **self.ws_defaults), partial(self.user_order_subscribe, symbol=s), self.message_handler, self.authenticate))
else:
ret.append((WSAsyncConn(self.address, self.id, **self.ws_defaults), self.subscribe, self.message_handler, self.authenticate))
return ret
async def update_subscription(self, subscription=None):
normalized_subscription = defaultdict(set)
for channel in subscription:
chan = feed_to_exchange(self.id, channel)
if is_authenticated_channel(channel):
if not self.key_id or not self.key_secret:
raise ValueError("Authenticated channel subscribed to, but no auth keys provided")
normalized_subscription[chan].update([self.std_symbol_to_exchange_symbol(symbol, self.id) for symbol in subscription[channel]])
channels_to_subscribe = []
channels_to_unsubscribe = []
for chan in normalized_subscription:
for pair in (set(normalized_subscription[chan]) - set(self.subscription[chan])):
channels_to_subscribe.append(Deribit.build_channel_name(chan, pair))
for pair in (set(self.subscription[chan]) - set(normalized_subscription[chan])):
channels_to_unsubscribe.append(Deribit.build_channel_name(chan, pair))
self.subscription = normalized_subscription
LOG.info(f"Updating subscription. Channels to subscribe: {channels_to_subscribe}. Channels to unsubscribe: {channels_to_unsubscribe}")
await self.subscribe_inner(self.connection, channels_to_subscribe)
await self.unsubscribe_inner(self.connection, channels_to_unsubscribe)
def __init__(self,
address: Union[dict, str],
timeout=120,
timeout_interval=30,
retries=10,
symbols=None,
channels=None,
subscription=None,
config: Union[Config, dict, str] = None,
callbacks=None,
max_depth=None,
book_interval=1000,
snapshot_interval=False,
checksum_validation=False,
cross_check=False,
origin=None,
exceptions=None,
log_message_on_error=False,
sandbox=False):
"""
address: str, or dict
address to be used to create the connection.
The address protocol (wss or https) will be used to determine the connection type.
Use a "str" to pass one single address, or a dict of option/address
timeout: int
Time, in seconds, between message to wait before a feed is considered dead and will be restarted.
Set to -1 for infinite.
timeout_interval: int
Time, in seconds, between timeout checks.
retries: int
Number of times to retry a failed connection. Set to -1 for infinite
max_depth: int
Maximum number of levels per side to return in book updates
book_interval: int
Number of updates between snapshots. Only applicable when book deltas are enabled.
Book deltas are enabled by subscribing to the book delta callback.
snapshot_interval: bool/int
Number of updates between snapshots. Only applicable when book delta is not enabled.
Updates between snapshots are not delivered to the client
checksum_validation: bool
Toggle checksum validation, when supported by an exchange.
cross_check: bool
Toggle a check for a crossed book. Should not be needed on exchanges that support
checksums or provide message sequence numbers.
origin: str
Passed into websocket connect. Sets the origin header.
exceptions: list of exceptions
These exceptions will not be handled internally and will be passed to the asyncio exception handler. To
handle them feedhandler will need to be supplied with a custom exception handler. See the `run` method
on FeedHandler, specifically the `exception_handler` keyword argument.
log_message_on_error: bool
If an exception is encountered in the connection handler, log the raw message
sandbox: bool
enable sandbox mode for exchanges that support this
"""
if isinstance(config, Config):
LOG.info(
'%s: reuse object Config containing the following main keys: %s',
self.id, ", ".join(config.config.keys()))
self.config = config
else:
LOG.info('%s: create Config from type: %r', self.id, type(config))
self.config = Config(config)
self.sandbox = sandbox
self.log_on_error = log_message_on_error
self.retries = retries
self.exceptions = exceptions
self.connection_handlers = []
self.timeout = timeout
self.timeout_interval = timeout_interval
self.subscription = defaultdict(set)
self.address = address
self.book_update_interval = book_interval
self.snapshot_interval = snapshot_interval
self.cross_check = cross_check
self.updates = defaultdict(int)
self.do_deltas = False
self.normalized_symbols = []
self.max_depth = max_depth
self.previous_book = defaultdict(dict)
self.origin = origin
self.checksum_validation = checksum_validation
self.ws_defaults = {
'ping_interval': 10,
'ping_timeout': None,
'max_size': 2**23,
'max_queue': None,
'origin': self.origin
}
self.key_id = os.environ.get(f'CF_{self.id}_KEY_ID') or self.config[
self.id.lower()].key_id
self.key_secret = os.environ.get(
f'CF_{self.id}_KEY_SECRET') or self.config[
self.id.lower()].key_secret
self._feed_config = defaultdict(list)
self.http_conn = HTTPAsyncConn(self.id)
symbols_cache = Symbols
if not symbols_cache.populated(self.id):
self.symbol_mapping()
self.normalized_symbol_mapping, self.exchange_info = symbols_cache.get(
self.id)
self.exchange_symbol_mapping = {
value: key
for key, value in self.normalized_symbol_mapping.items()
}
if subscription is not None and (symbols is not None
or channels is not None):
raise ValueError(
"Use subscription, or channels and symbols, not both")
if subscription is not None:
self.channels = list(subscription.keys())
self.symbols = list(subscription[self.channels[0]])
for channel in subscription:
chan = feed_to_exchange(self.id, channel)
if is_authenticated_channel(channel):
if not self.key_id or not self.key_secret:
raise ValueError(
"Authenticated channel subscribed to, but no auth keys provided"
)
self.normalized_symbols.extend(subscription[channel])
self.subscription[chan].update([
self.std_symbol_to_exchange_symbol(symbol)
for symbol in subscription[channel]
])
self._feed_config[channel].extend(self.normalized_symbols)
if symbols and channels:
if any(is_authenticated_channel(chan) for chan in channels):
if not self.key_id or not self.key_secret:
raise ValueError(
"Authenticated channel subscribed to, but no auth keys provided"
)
self.channels = channels
self.symbols = symbols
# if we dont have a subscription dict, we'll use symbols+channels and build one
[
self._feed_config[channel].extend(symbols)
for channel in channels
]
self.normalized_symbols = symbols
symbols = [
self.std_symbol_to_exchange_symbol(symbol)
for symbol in symbols
]
channels = list(
set([feed_to_exchange(self.id, chan) for chan in channels]))
self.subscription = {chan: symbols for chan in channels}
self._feed_config = dict(self._feed_config)
self.l3_book = {}
self.l2_book = {}
self.callbacks = {
FUNDING: Callback(None),
FUTURES_INDEX: Callback(None),
L2_BOOK: Callback(None),
L3_BOOK: Callback(None),
LIQUIDATIONS: Callback(None),
OPEN_INTEREST: Callback(None),
MARKET_INFO: Callback(None),
TICKER: Callback(None),
TRADES: Callback(None),
CANDLES: Callback(None),
ORDER_INFO: Callback(None)
}
if callbacks:
for cb_type, cb_func in callbacks.items():
self.callbacks[cb_type] = cb_func
if cb_type == BOOK_DELTA:
self.do_deltas = True
for key, callback in self.callbacks.items():
if not isinstance(callback, list):
self.callbacks[key] = [callback]
def __init__(self,
address: Union[dict, str],
sandbox=False,
symbols=None,
channels=None,
subscription=None,
config: Union[Config, dict, str] = None,
callbacks=None,
max_depth=None,
book_interval=1000,
snapshot_interval=False,
checksum_validation=False,
cross_check=False,
origin=None):
"""
address: str, or dict
address to be used to create the connection.
The address protocol (wss or https) will be used to determine the connection type.
Use a "str" to pass one single address, or a dict of option/address
sandbox: bool
For authenticated channels, run against the sandbox websocket (when True)
max_depth: int
Maximum number of levels per side to return in book updates
book_interval: int
Number of updates between snapshots. Only applicable when book deltas are enabled.
Book deltas are enabled by subscribing to the book delta callback.
snapshot_interval: bool/int
Number of updates between snapshots. Only applicable when book delta is not enabled.
Updates between snapshots are not delivered to the client
checksum_validation: bool
Toggle checksum validation, when supported by an exchange.
cross_check: bool
Toggle a check for a crossed book. Should not be needed on exchanges that support
checksums or provide message sequence numbers.
origin: str
Passed into websocket connect. Sets the origin header.
"""
if isinstance(config, Config):
LOG.info(
'%s: reuse object Config containing the following main keys: %s',
self.id, ", ".join(config.config.keys()))
self.config = config
else:
LOG.info('%s: create Config from type: %r', self.id, type(config))
self.config = Config(config)
self.subscription = defaultdict(set)
self.address = address
self.book_update_interval = book_interval
self.snapshot_interval = snapshot_interval
self.cross_check = cross_check
self.updates = defaultdict(int)
self.do_deltas = False
self.symbols = []
self.normalized_symbols = []
self.channels = []
self.max_depth = max_depth
self.previous_book = defaultdict(dict)
self.origin = origin
self.checksum_validation = checksum_validation
self.ws_defaults = {
'ping_interval': 10,
'ping_timeout': None,
'max_size': 2**23,
'max_queue': None,
'origin': self.origin
}
self.key_id = os.environ.get(f'CF_{self.id}_KEY_ID') or self.config[
self.id.lower()].key_id
self.key_secret = os.environ.get(
f'CF_{self.id}_KEY_SECRET') or self.config[
self.id.lower()].key_secret
self._feed_config = defaultdict(list)
load_exchange_symbol_mapping(self.id, key_id=self.key_id)
if subscription is not None and (symbols is not None
or channels is not None):
raise ValueError(
"Use subscription, or channels and symbols, not both")
if subscription is not None:
for channel in subscription:
chan = feed_to_exchange(self.id, channel)
if is_authenticated_channel(channel):
if not self.key_id or not self.key_secret:
raise ValueError(
"Authenticated channel subscribed to, but no auth keys provided"
)
self.normalized_symbols.extend(subscription[channel])
self.subscription[chan].update([
symbol_std_to_exchange(symbol, self.id)
for symbol in subscription[channel]
])
self._feed_config[channel].extend(self.normalized_symbols)
if symbols:
self.normalized_symbols = symbols
self.symbols = [
symbol_std_to_exchange(symbol, self.id) for symbol in symbols
]
if channels:
self.channels = list(
set([feed_to_exchange(self.id, chan) for chan in channels]))
[
self._feed_config[channel].extend(self.normalized_symbols)
for channel in channels
]
if any(is_authenticated_channel(chan) for chan in channels):
if not self.key_id or not self.key_secret:
raise ValueError(
"Authenticated channel subscribed to, but no auth keys provided"
)
self._feed_config = dict(self._feed_config)
self.l3_book = {}
self.l2_book = {}
self.callbacks = {
FUNDING: Callback(None),
FUTURES_INDEX: Callback(None),
L2_BOOK: Callback(None),
L3_BOOK: Callback(None),
LIQUIDATIONS: Callback(None),
OPEN_INTEREST: Callback(None),
MARKET_INFO: Callback(None),
TICKER: Callback(None),
TRADES: Callback(None),
TRANSACTIONS: Callback(None),
VOLUME: Callback(None),
CANDLES: Callback(None),
ORDER_INFO: Callback(None)
}
if callbacks:
for cb_type, cb_func in callbacks.items():
self.callbacks[cb_type] = cb_func
if cb_type == BOOK_DELTA:
self.do_deltas = True
for key, callback in self.callbacks.items():
if not isinstance(callback, list):
self.callbacks[key] = [callback]
