class BinanceUS(Binance, BinanceUSRestMixin):
id = BINANCE_US
websocket_endpoints = [WebsocketEndpoint('wss://stream.binance.us:9443')]
rest_endpoints = [
RestEndpoint('https://api.binance.us',
routes=Routes('/api/v3/exchangeInfo',
l2book='/api/v3/depth?symbol={}&limit={}'))
]
class FTXTR(FTX, FTXTRRestMixin):
id = FTX_TR
websocket_endpoints = [WebsocketEndpoint("wss://ftxtr.com/ws/", options={"compression": None})]
rest_endpoints = [RestEndpoint("https://ftxtr.com", routes=Routes("/api/markets"))]
websocket_channels = {
L2_BOOK: "orderbook",
TRADES: "trades",
TICKER: "ticker",
ORDER_INFO: "orders",
FILLS: "fills",
}
class FTXUS(FTX, FTXUSRestMixin):
id = FTX_US
websocket_endpoints = [
WebsocketEndpoint('wss://ftx.us/ws/', options={'compression': None})
]
rest_endpoints = [
RestEndpoint('https://ftx.us', routes=Routes('/api/markets'))
]
websocket_channels = {
L2_BOOK: 'orderbook',
TRADES: 'trades',
TICKER: 'ticker',
ORDER_INFO: 'orders',
FILLS: 'fills',
}
class HitBTC(Bequant):
id = HITBTC
websocket_channels = {
BALANCES: 'subscribeBalance',
TRANSACTIONS: 'subscribeTransactions',
ORDER_INFO: 'subscribeReports',
L2_BOOK: 'subscribeOrderbook',
TRADES: 'subscribeTrades',
TICKER: 'subscribeTicker',
CANDLES: 'subscribeCandles'
}
websocket_endpoints = [
WebsocketEndpoint('wss://api.hitbtc.com/api/2/ws/public', channel_filter=(websocket_channels[L2_BOOK], websocket_channels[TRADES], websocket_channels[TICKER], websocket_channels[CANDLES])),
WebsocketEndpoint('wss://api.hitbtc.com/api/2/ws/trading', channel_filter=(websocket_channels[ORDER_INFO],)),
WebsocketEndpoint('wss://api.hitbtc.com/api/2/ws/account', channel_filter=(websocket_channels[BALANCES], websocket_channels[TRANSACTIONS])),
]
rest_endpoints = [RestEndpoint('https://api.hitbtc.com', routes=Routes('/api/2/public/symbol'))]
class AscendEXFutures(AscendEX):
"""
Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#introducing-futures-pro-v2-apis
"""
id = ASCENDEX_FUTURES
websocket_channels = {
**AscendEX.websocket_channels,
}
# Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#how-to-connect
# noinspection PyTypeChecker
websocket_endpoints = [
WebsocketEndpoint(
'wss://ascendex.com:443/api/pro/v2/stream',
channel_filter=(
websocket_channels[L2_BOOK],
websocket_channels[TRADES],
),
sandbox='wss://api-test.ascendex-sandbox.com:443/api/pro/v2/stream'
)
]
# Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#futures-contracts-info
rest_endpoints = [
RestEndpoint('https://ascendex.com',
routes=Routes('/api/pro/v2/futures/contract'),
sandbox='https://api-test.ascendex-sandbox.com')
]
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
# Docs, https://ascendex.github.io/ascendex-futures-pro-api-v2/#futures-contracts-info
ret = {}
info = defaultdict(dict)
for entry in data['data']:
# Only "Normal" status symbols are tradeable
if entry['status'] == 'Normal':
s = Symbol(re.sub(entry['settlementAsset'], '',
entry['displayName']),
entry['settlementAsset'],
type=PERPETUAL)
ret[s.normalized] = entry['symbol']
info['tick_size'][
s.normalized] = entry['priceFilter']['tickSize']
info['instrument_type'][s.normalized] = s.type
return ret, info
class IndependentReserve(Feed):
id = INDEPENDENT_RESERVE
websocket_endpoints = [WebsocketEndpoint('wss://websockets.independentreserve.com')]
rest_endpoints = [RestEndpoint('https://api.independentreserve.com', routes=Routes(['/Public/GetValidPrimaryCurrencyCodes', '/Public/GetValidSecondaryCurrencyCodes'], l3book='/Public/GetAllOrders?primaryCurrencyCode={}&secondaryCurrencyCode={}'))]
websocket_channels = {
L3_BOOK: 'orderbook-{}',
TRADES: 'ticker-{}',
}
request_limit = 1
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
bases, quotes = data
for base in bases:
for quote in quotes:
sym = Symbol(base.upper().replace('XBT', 'BTC'), quote.upper())
info['instrument_type'][sym.normalized] = sym.type
ret[sym.normalized] = f"{base.lower()}-{quote.lower()}"
return ret, info
def __reset(self):
self._l3_book = {}
self._order_ids = defaultdict(dict)
self._sequence_no = {}
async def _trade(self, msg: dict, timestamp: float):
'''
{
'Channel': 'ticker-eth-aud',
'Nonce': 78,
'Data': {
'TradeGuid': '6d1c2e90-592a-409c-a8d8-58b2d25e0b0b',
'Pair': 'eth-aud',
'TradeDate': datetime.datetime(2022, 1, 31, 8, 28, 26, 552573, tzinfo=datetime.timezone(datetime.timedelta(seconds=39600))),
'Price': Decimal('3650.81'),
'Volume': Decimal('0.543'),
'BidGuid': '0430e003-c35e-410e-85f5-f0bb5c40193b',
'OfferGuid': '559c1dd2-e681-4efc-b49b-14a07c069de4',
'Side': 'Sell'
},
'Time': 1643578106584,
'Event': 'Trade'
}
'''
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(msg['Data']['Pair']),
SELL if msg['Data']['Side'] == 'Sell' else BUY,
Decimal(msg['Data']['Volume']),
Decimal(msg['Data']['Price']),
self.timestamp_normalize(msg['Data']['TradeDate']),
id=msg['Data']['TradeGuid'],
raw=msg
)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
'''
{
'Channel': 'orderbook-xbt',
'Nonce': 65605,
'Data': {
'OrderType': 'LimitBid',
'OrderGuid': 'fee7094c-1921-44b7-8d8d-8b6e1cedb270'
},
'Time': 1643931382903,
'Event': 'OrderCanceled'
}
{
'Channel': 'orderbook-xbt',
'Nonce': 65606,
'Data': {
'OrderType': 'LimitOffer',
'OrderGuid': '22a72137-9829-4e6c-b265-a38714256877',
'Price': {
'aud': Decimal('51833.41'),
'usd': Decimal('37191.23'),
'nzd': Decimal('55836.92'),
'sgd': Decimal('49892.59')
},
'Volume': Decimal('0.09')
},
'Time': 1643931382903,
'Event': 'NewOrder'
}
'''
seq_no = msg['Nonce']
base = msg['Channel'].split('-')[-1]
delta = {BID: [], ASK: []}
for symbol in self.subscription[self.std_channel_to_exchange(L3_BOOK)]:
if symbol.startswith(base):
quote = symbol.split('-')[-1]
instrument = self.exchange_symbol_to_std_symbol(f"{base}-{quote}")
if instrument in self._sequence_no and self._sequence_no[instrument] + 1 != seq_no:
raise MissingSequenceNumber
self._sequence_no[instrument] = seq_no
if instrument not in self._l3_book:
await self._snapshot(base, quote)
if msg['Event'] == 'OrderCanceled':
uuid = msg['Data']['OrderGuid']
if uuid in self._order_ids[instrument]:
price, side = self._order_ids[instrument][uuid]
if price in self._l3_book[instrument].book[side] and uuid in self._l3_book[instrument].book[side][price]:
del self._l3_book[instrument].book[side][price][uuid]
if len(self._l3_book[instrument].book[side][price]) == 0:
del self._l3_book[instrument].book[side][price]
delta[side].append((uuid, price, 0))
del self._order_ids[instrument][uuid]
else:
# during snapshots we might get cancelation messages that have already been removed
# from the snapshot, so we don't have anything to process, and we should not call the client callback
continue
elif msg['Event'] == 'NewOrder':
uuid = msg['Data']['OrderGuid']
price = msg['Data']['Price'][quote]
size = msg['Data']['Volume']
side = BID if msg['Data']['OrderType'].endswith('Bid') else ASK
self._order_ids[instrument][uuid] = (price, side)
if price in self._l3_book[instrument].book[side]:
self._l3_book[instrument].book[side][price][uuid] = size
else:
self._l3_book[instrument].book[side][price] = {uuid: size}
delta[side].append((uuid, price, size))
elif msg['event'] == 'OrderChanged':
uuid = msg['Data']['OrderGuid']
size = msg['Data']['Volume']
side = BID if msg['Data']['OrderType'].endswith('Bid') else ASK
if uuid in self._order_ids[instrument]:
price, side = self._order_ids[instrument][uuid]
if size == 0:
del self._l3_book[instrument][side][price][uuid]
if len(self._l3_book[instrument][side][price]) == 0:
del self._l3_book[instrument][side][price]
else:
self._l3_book[instrument][side][price][uuid] = size
del self._order_ids[instrument][uuid]
delta[side].append((uuid, price, size))
else:
continue
else:
raise ValueError("%s: Invalid OrderBook event message of type %s", self.id, msg)
await self.book_callback(L3_BOOK, self._l3_book[instrument], timestamp, raw=msg, sequence_number=seq_no, delta=delta, timestamp=msg['Time'] / 1000)
async def _snapshot(self, base: str, quote: str):
url = self.rest_endpoints[0].route('l3book', self.sandbox).format(base, quote)
timestamp = time()
ret = await self.http_conn.read(url)
await asyncio.sleep(1 / self.request_limit)
ret = json.loads(ret, parse_float=Decimal)
normalized = self.exchange_symbol_to_std_symbol(f"{base}-{quote}")
self._l3_book[normalized] = OrderBook(self.id, normalized, max_depth=self.max_depth)
for side, key in [(BID, 'BuyOrders'), (ASK, 'SellOrders')]:
for order in ret[key]:
price = Decimal(order['Price'])
size = Decimal(order['Volume'])
uuid = order['Guid']
self._order_ids[normalized][uuid] = (price, side)
if price in self._l3_book[normalized].book[side]:
self._l3_book[normalized].book[side][price][uuid] = size
else:
self._l3_book[normalized].book[side][price] = {uuid: size}
await self.book_callback(L3_BOOK, self._l3_book[normalized], timestamp, raw=ret)
async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['Event'] == 'Trade':
await self._trade(msg, timestamp)
elif msg['Event'] in ('OrderCanceled', 'OrderChanged', 'NewOrder'):
await self._book(msg, timestamp)
elif msg['Event'] in ('Subscriptions', 'Heartbeat', 'Unsubscribe'):
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
subs = []
for chan, symbols in conn.subscription.items():
if self.exchange_channel_to_std(chan) == L3_BOOK:
subs.extend([chan.format(s) for s in set([sym.split("-")[0] for sym in symbols])])
else:
subs.extend([chan.format(s) for s in symbols])
await conn.write(json.dumps({"Event": "Subscribe", "Data": subs}))
class FTX(Feed, FTXRestMixin):
id = FTX_id
websocket_endpoints = [
WebsocketEndpoint('wss://ftx.com/ws/', options={'compression': None})
]
rest_endpoints = [
RestEndpoint('https://ftx.com',
routes=Routes('/api/markets',
open_interest='/api/futures/{}/stats',
funding='/api/funding_rates?future={}',
stats='/api/futures/{}/stats'))
]
websocket_channels = {
L2_BOOK: 'orderbook',
TRADES: 'trades',
TICKER: 'ticker',
FUNDING: 'funding',
OPEN_INTEREST: 'open_interest',
LIQUIDATIONS: 'trades',
ORDER_INFO: 'orders',
FILLS: 'fills',
}
request_limit = 30
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for d in data['result']:
if not d['enabled']:
continue
expiry = None
stype = SPOT
# FTX Futures contracts are stable coin settled, but
# prices quoted are in USD, see https://help.ftx.com/hc/en-us/articles/360024780791-What-Are-Futures
if "-MOVE-" in d['name']:
stype = FUTURES
base, expiry = d['name'].rsplit("-", maxsplit=1)
quote = 'USD'
if 'Q' in expiry:
year, quarter = expiry.split("Q")
year = year[2:]
date = ["0325", "0624", "0924", "1231"]
expiry = year + date[int(quarter) - 1]
elif "-" in d['name']:
base, expiry = d['name'].split("-")
quote = 'USD'
stype = FUTURES
if expiry == 'PERP':
expiry = None
stype = PERPETUAL
elif d['type'] == SPOT:
base, quote = d['baseCurrency'], d['quoteCurrency']
else:
# not enough info to construct a symbol - this is usually caused
# by non crypto futures, i.e. TRUMP2024 or other contracts involving
# betting on world events
continue
s = Symbol(base, quote, type=stype, expiry_date=expiry)
ret[s.normalized] = d['name']
info['tick_size'][s.normalized] = d['priceIncrement']
info['quantity_step'][s.normalized] = d['sizeIncrement']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
self._funding_cache = {}
self._open_interest_cache = {}
async def generate_token(self, conn: AsyncConnection):
ts = int(time() * 1000)
msg = {
'op': 'login',
'args': {
'key':
self.key_id,
'sign':
hmac.new(self.key_secret.encode(),
f'{ts}websocket_login'.encode(),
'sha256').hexdigest(),
'time':
ts,
}
}
if self.subaccount:
msg['args']['subaccount'] = self.subaccount
await conn.write(json.dumps(msg))
async def authenticate(self, conn: AsyncConnection):
if self.requires_authentication:
await self.generate_token(conn)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
symbols = self.subscription[chan]
if chan == FUNDING:
asyncio.create_task(self._funding(symbols))
continue
if chan == OPEN_INTEREST:
asyncio.create_task(self._open_interest(symbols))
continue
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
await conn.write(
json.dumps({
"channel": chan,
"op": "subscribe"
}))
continue
for pair in symbols:
await conn.write(
json.dumps({
"channel": chan,
"market": pair,
"op": "subscribe"
}))
async def _open_interest(self, pairs: Iterable):
"""
{
"success": true,
"result": {
"volume": 1000.23,
"nextFundingRate": 0.00025,
"nextFundingTime": "2019-03-29T03:00:00+00:00",
"expirationPrice": 3992.1,
"predictedExpirationPrice": 3993.6,
"strikePrice": 8182.35,
"openInterest": 21124.583
}
}
"""
while True:
for pair in pairs:
# OI only for perp and futures, so check for / in pair name indicating spot
if '/' in pair:
continue
data = await self.http_conn.read(self.rest_endpoints[0].route(
'open_interest', sandbox=self.sandbox).format(pair))
received = time()
data = json.loads(data, parse_float=Decimal)
if 'result' in data:
oi = data['result']['openInterest']
if oi != self._open_interest_cache.get(pair, None):
o = OpenInterest(
self.id,
self.exchange_symbol_to_std_symbol(pair),
oi,
None,
raw=data)
await self.callback(OPEN_INTEREST, o, received)
self._open_interest_cache[pair] = oi
await asyncio.sleep(1)
await asyncio.sleep(60)
async def _funding(self, pairs: Iterable):
"""
{
"success": true,
"result": [
{
"future": "BTC-PERP",
"rate": 0.0025,
"time": "2019-06-02T08:00:00+00:00"
}
]
}
"""
while True:
for pair in pairs:
if '-PERP' not in pair:
continue
data = await self.http_conn.read(self.rest_endpoints[0].route(
'funding', sandbox=self.sandbox).format(pair))
data = json.loads(data, parse_float=Decimal)
data2 = await self.http_conn.read(self.rest_endpoints[0].route(
'stats', sandbox=self.sandbox).format(pair))
data2 = json.loads(data2, parse_float=Decimal)
received = time()
data['predicted_rate'] = Decimal(
data2['result']['nextFundingRate'])
last_update = self._funding_cache.get(pair, None)
update = str(data['result'][0]['rate']) + str(
data['result'][0]['time']) + str(data['predicted_rate'])
if last_update and last_update == update:
continue
else:
self._funding_cache[pair] = update
f = Funding(self.id,
self.exchange_symbol_to_std_symbol(
data['result'][0]['future']),
None,
data['result'][0]['rate'],
self.timestamp_normalize(
data2['result']['nextFundingTime']),
self.timestamp_normalize(
data['result'][0]['time']),
predicted_rate=data['predicted_rate'],
raw=[data, data2])
await self.callback(FUNDING, f, received)
await asyncio.sleep(0.1)
await asyncio.sleep(60)
async def _trade(self, msg: dict, timestamp: float):
"""
example message:
{"channel": "trades", "market": "BTC-PERP", "type": "update", "data": [{"id": null, "price": 10738.75,
"size": 0.3616, "side": "buy", "liquidation": false, "time": "2019-08-03T12:20:19.170586+00:00"}]}
"""
for trade in msg['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['market']),
BUY if trade['side'] == 'buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
float(self.timestamp_normalize(trade['time'])),
id=str(trade['id']),
raw=trade)
await self.callback(TRADES, t, timestamp)
if bool(trade['liquidation']):
liq = Liquidation(
self.id,
self.exchange_symbol_to_std_symbol(msg['market']),
BUY if trade['side'] == 'buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
str(trade['id']),
FILLED,
float(self.timestamp_normalize(trade['time'])),
raw=trade)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def _ticker(self, msg: dict, timestamp: float):
"""
example message:
{"channel": "ticker", "market": "BTC/USD", "type": "update", "data": {"bid": 10717.5, "ask": 10719.0,
"last": 10719.0, "time": 1564834587.1299787}}
"""
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(msg['market']),
Decimal(msg['data']['bid'] if msg['data']['bid'] else 0.0),
Decimal(msg['data']['ask'] if msg['data']['ask'] else 0.0),
float(msg['data']['time']),
raw=msg)
await self.callback(TICKER, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
"""
example messages:
snapshot:
{"channel": "orderbook", "market": "BTC/USD", "type": "partial", "data": {"time": 1564834586.3382702,
"checksum": 427503966, "bids": [[10717.5, 4.092], ...], "asks": [[10720.5, 15.3458], ...], "action": "partial"}}
update:
{"channel": "orderbook", "market": "BTC/USD", "type": "update", "data": {"time": 1564834587.1299787,
"checksum": 3115602423, "bids": [], "asks": [[10719.0, 14.7461]], "action": "update"}}
"""
check = msg['data']['checksum']
if msg['type'] == 'partial':
# snapshot
pair = self.exchange_symbol_to_std_symbol(msg['market'])
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth,
checksum_format='FTX')
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for price, amount in msg['data']['bids']
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for price, amount in msg['data']['asks']
}
if self.checksum_validation and self._l2_book[pair].book.checksum(
) != check:
raise BadChecksum
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=float(msg['data']['time']),
raw=msg,
checksum=check)
else:
# update
delta = {BID: [], ASK: []}
pair = self.exchange_symbol_to_std_symbol(msg['market'])
for side in ('bids', 'asks'):
s = BID if side == 'bids' else ASK
for price, amount in msg['data'][side]:
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
delta[s].append((price, 0))
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, amount))
self._l2_book[pair].book[s][price] = amount
if self.checksum_validation and self._l2_book[pair].book.checksum(
) != check:
raise BadChecksum
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=float(msg['data']['time']),
raw=msg,
checksum=check,
delta=delta)
async def _fill(self, msg: dict, timestamp: float):
"""
example message:
{
"channel": "fills",
"data": {
"fee": 78.05799225,
"feeRate": 0.0014,
"future": "BTC-PERP",
"id": 7828307,
"liquidity": "taker",
"market": "BTC-PERP",
"orderId": 38065410,
"tradeId": 19129310,
"price": 3723.75,
"side": "buy",
"size": 14.973,
"time": "2019-05-07T16:40:58.358438+00:00",
"type": "order"
},
"type": "update"
}
"""
fill = msg['data']
f = Fill(self.id,
self.exchange_symbol_to_std_symbol(fill['market']),
BUY if fill['side'] == 'buy' else SELL,
Decimal(fill['size']),
Decimal(fill['price']),
Decimal(fill['fee']),
str(fill['tradeId']),
str(fill['orderId']),
None,
TAKER if fill['liquidity'] == 'taker' else MAKER,
fill['time'].timestamp(),
account=self.subaccount,
raw=msg)
await self.callback(FILLS, f, timestamp)
async def _order(self, msg: dict, timestamp: float):
"""
example message:
{
"channel": "orders",
"data": {
"id": 24852229,
"clientId": null,
"market": "XRP-PERP",
"type": "limit",
"side": "buy",
"size": 42353.0,
"price": 0.2977,
"reduceOnly": false,
"ioc": false,
"postOnly": false,
"status": "closed",
"filledSize": 0.0,
"remainingSize": 0.0,
"avgFillPrice": 0.2978
},
"type": "update"
}
"""
order = msg['data']
status = order['status']
if status == 'new':
status = SUBMITTING
elif status == 'open':
status = OPEN
elif status == 'closed':
status = CLOSED
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(order['market']),
str(order['id']),
BUY if order['side'].lower() == 'buy' else SELL,
status,
LIMIT if order['type'] == 'limit' else MARKET,
Decimal(order['price']) if order['price'] else None,
Decimal(order['filledSize']),
Decimal(order['remainingSize']),
None,
account=self.subaccount,
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'type' in msg:
if msg['type'] == 'subscribed':
if 'market' in msg:
LOG.info('%s: Subscribed to %s channel for %s', self.id,
msg['channel'], msg['market'])
else:
LOG.info('%s: Subscribed to %s channel', self.id,
msg['channel'])
elif msg['type'] == 'error':
LOG.error('%s: Received error message %s', self.id, msg)
raise Exception('Error from %s: %s', self.id, msg)
elif 'channel' in msg:
if msg['channel'] == 'orderbook':
await self._book(msg, timestamp)
elif msg['channel'] == 'trades':
await self._trade(msg, timestamp)
elif msg['channel'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['channel'] == 'fills':
await self._fill(msg, timestamp)
elif msg['channel'] == 'orders':
await self._order(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
class Bybit(Feed):
id = BYBIT
websocket_channels = {
# TODO make aggregation window (100ms) configurable
# TODO figure out ValueError: Use subscription, or channels and symbols, not both
# TODO OPEN_INTEREST and FUNDING report INDEX data_type
L2_BOOK: 'orderBook_200.100ms',
TRADES: 'trade',
FILLS: 'execution',
ORDER_INFO: 'order',
INDEX: 'instrument_info.100ms',
OPEN_INTEREST: 'instrument_info.100ms',
FUNDING: 'instrument_info.100ms',
CANDLES: 'klineV2',
LIQUIDATIONS: 'liquidation'
}
websocket_endpoints = [
WebsocketEndpoint('wss://stream.bybit.com/realtime', channel_filter=(websocket_channels[L2_BOOK], websocket_channels[TRADES], websocket_channels[INDEX], websocket_channels[OPEN_INTEREST], websocket_channels[FUNDING], websocket_channels[CANDLES], websocket_channels[LIQUIDATIONS]), instrument_filter=('QUOTE', ('USD',)), sandbox='wss://stream-testnet.bybit.com/realtime', options={'compression': None}),
WebsocketEndpoint('wss://stream.bybit.com/realtime_public', channel_filter=(websocket_channels[L2_BOOK], websocket_channels[TRADES], websocket_channels[INDEX], websocket_channels[OPEN_INTEREST], websocket_channels[FUNDING], websocket_channels[CANDLES], websocket_channels[LIQUIDATIONS]), instrument_filter=('QUOTE', ('USDT',)), sandbox='wss://stream-testnet.bybit.com/realtime_public', options={'compression': None}),
WebsocketEndpoint('wss://stream.bybit.com/realtime_private', channel_filter=(websocket_channels[ORDER_INFO], websocket_channels[FILLS]), instrument_filter=('QUOTE', ('USDT',)), sandbox='wss://stream-testnet.bybit.com/realtime_private', options={'compression': None}),
]
rest_endpoints = [RestEndpoint('https://api.bybit.com', routes=Routes('/v2/public/symbols'))]
valid_candle_intervals = {'1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '1d', '1w', '1M'}
candle_interval_map = {'1m': '1', '3m': '3', '5m': '5', '15m': '15', '30m': '30', '1h': '60', '2h': '120', '4h': '240', '6h': '360', '1d': 'D', '1w': 'W', '1M': 'M'}
@classmethod
def timestamp_normalize(cls, ts: Union[int, dt]) -> float:
if isinstance(ts, int):
return ts / 1000.0
else:
return ts.timestamp()
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
# PERPETUAL & FUTURES
for symbol in data['result']:
base = symbol['base_currency']
quote = symbol['quote_currency']
stype = PERPETUAL
expiry = None
if not symbol['name'].endswith(quote):
stype = FUTURES
year = symbol['name'].replace(base + quote, '')[-2:]
expiry = year + symbol['alias'].replace(base + quote, '')[-4:]
s = Symbol(base, quote, type=stype, expiry_date=expiry)
ret[s.normalized] = symbol['name']
info['tick_size'][s.normalized] = symbol['price_filter']['tick_size']
info['instrument_type'][s.normalized] = stype
return ret, info
def __reset(self):
self._instrument_info_cache = {}
self._l2_book = {}
async def _candle(self, msg: dict, timestamp: float):
'''
{
"topic": "klineV2.1.BTCUSD", //topic name
"data": [{
"start": 1572425640, //start time of the candle
"end": 1572425700, //end time of the candle
"open": 9200, //open price
"close": 9202.5, //close price
"high": 9202.5, //max price
"low": 9196, //min price
"volume": 81790, //volume
"turnover": 8.889247899999999, //turnover
"confirm": False, //snapshot flag (indicates if candle is closed or not)
"cross_seq": 297503466,
"timestamp": 1572425676958323 //cross time
}],
"timestamp_e6": 1572425677047994 //server time
}
'''
symbol = self.exchange_symbol_to_std_symbol(msg['topic'].split(".")[-1])
ts = msg['timestamp_e6'] / 1_000_000
for entry in msg['data']:
if self.candle_closed_only and not entry['confirm']:
continue
c = Candle(self.id,
symbol,
entry['start'],
entry['end'],
self.candle_interval,
entry['confirm'],
Decimal(entry['open']),
Decimal(entry['close']),
Decimal(entry['high']),
Decimal(entry['low']),
Decimal(entry['volume']),
None,
ts,
raw=entry)
await self.callback(CANDLES, c, timestamp)
async def _liquidation(self, msg: dict, timestamp: float):
'''
{
'topic': 'liquidation.EOSUSDT',
'data': {
'symbol': 'EOSUSDT',
'side': 'Buy',
'price': '3.950',
'qty': '58.0',
'time': 1632586154956
}
}
'''
liq = Liquidation(
self.id,
self.exchange_symbol_to_std_symbol(msg['data']['symbol']),
BUY if msg['data']['side'] == 'Buy' else SELL,
Decimal(msg['data']['qty']),
Decimal(msg['data']['price']),
None,
None,
self.timestamp_normalize(msg['data']['time']),
raw=msg
)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if "success" in msg:
if msg['success']:
if msg['request']['op'] == 'auth':
LOG.debug("%s: Authenticated successful", conn.uuid)
elif msg['request']['op'] == 'subscribe':
LOG.debug("%s: Subscribed to channels: %s", conn.uuid, msg['request']['args'])
else:
LOG.warning("%s: Unhandled 'successs' message received", conn.uuid)
else:
LOG.error("%s: Error from exchange %s", conn.uuid, msg)
elif msg["topic"].startswith('trade'):
await self._trade(msg, timestamp)
elif msg["topic"].startswith('orderBook'):
await self._book(msg, timestamp)
elif msg['topic'].startswith('liquidation'):
await self._liquidation(msg, timestamp)
elif "instrument_info" in msg["topic"]:
await self._instrument_info(msg, timestamp)
elif "order" in msg["topic"]:
await self._order(msg, timestamp)
elif "execution" in msg["topic"]:
await self._execution(msg, timestamp)
elif 'klineV2' in msg['topic'] or 'candle' in msg['topic']:
await self._candle(msg, timestamp)
# elif "position" in msg["topic"]:
# await self._balances(msg, timestamp)
else:
LOG.warning("%s: Unhandled message type %s", conn.uuid, msg)
async def subscribe(self, connection: AsyncConnection):
self.__reset()
for chan in connection.subscription:
if not self.is_authenticated_channel(self.exchange_channel_to_std(chan)):
for pair in connection.subscription[chan]:
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(pair))
if self.exchange_channel_to_std(chan) == CANDLES:
c = chan if sym.quote == 'USD' else 'candle'
sub = [f"{c}.{self.candle_interval_map[self.candle_interval]}.{pair}"]
else:
sub = [f"{chan}.{pair}"]
await connection.write(json.dumps({"op": "subscribe", "args": sub}))
else:
await connection.write(json.dumps(
{
"op": "subscribe",
"args": [f"{chan}"]
}
))
async def _instrument_info(self, msg: dict, timestamp: float):
"""
### Snapshot type update
{
"topic": "instrument_info.100ms.BTCUSD",
"type": "snapshot",
"data": {
"id": 1,
"symbol": "BTCUSD", //instrument name
"last_price_e4": 81165000, //the latest price
"last_tick_direction": "ZeroPlusTick", //the direction of last tick:PlusTick,ZeroPlusTick,MinusTick,
//ZeroMinusTick
"prev_price_24h_e4": 81585000, //the price of prev 24h
"price_24h_pcnt_e6": -5148, //the current last price percentage change from prev 24h price
"high_price_24h_e4": 82900000, //the highest price of prev 24h
"low_price_24h_e4": 79655000, //the lowest price of prev 24h
"prev_price_1h_e4": 81395000, //the price of prev 1h
"price_1h_pcnt_e6": -2825, //the current last price percentage change from prev 1h price
"mark_price_e4": 81178500, //mark price
"index_price_e4": 81172800, //index price
"open_interest": 154418471, //open interest quantity - Attention, the update is not
//immediate - slowest update is 1 minute
"open_value_e8": 1997561103030, //open value quantity - Attention, the update is not
//immediate - the slowest update is 1 minute
"total_turnover_e8": 2029370141961401, //total turnover
"turnover_24h_e8": 9072939873591, //24h turnover
"total_volume": 175654418740, //total volume
"volume_24h": 735865248, //24h volume
"funding_rate_e6": 100, //funding rate
"predicted_funding_rate_e6": 100, //predicted funding rate
"cross_seq": 1053192577, //sequence
"created_at": "2018-11-14T16:33:26Z",
"updated_at": "2020-01-12T18:25:16Z",
"next_funding_time": "2020-01-13T00:00:00Z", //next funding time
//the rest time to settle funding fee
"countdown_hour": 6 //the remaining time to settle the funding fee
},
"cross_seq": 1053192634,
"timestamp_e6": 1578853524091081 //the timestamp when this information was produced
}
### Delta type update
{
"topic": "instrument_info.100ms.BTCUSD",
"type": "delta",
"data": {
"delete": [],
"update": [
{
"id": 1,
"symbol": "BTCUSD",
"prev_price_24h_e4": 81565000,
"price_24h_pcnt_e6": -4904,
"open_value_e8": 2000479681106,
"total_turnover_e8": 2029370495672976,
"turnover_24h_e8": 9066215468687,
"volume_24h": 735316391,
"cross_seq": 1053192657,
"created_at": "2018-11-14T16:33:26Z",
"updated_at": "2020-01-12T18:25:25Z"
}
],
"insert": []
},
"cross_seq": 1053192657,
"timestamp_e6": 1578853525691123
}
"""
update_type = msg['type']
if update_type == 'snapshot':
updates = [msg['data']]
else:
updates = msg['data']['update']
for info in updates:
if msg['topic'] in self._instrument_info_cache and self._instrument_info_cache[msg['topic']] == updates:
continue
else:
self._instrument_info_cache[msg['topic']] = updates
ts = int(msg['timestamp_e6']) / 1_000_000
if 'open_interest' in info:
oi = OpenInterest(
self.id,
self.exchange_symbol_to_std_symbol(info['symbol']),
Decimal(info['open_interest']),
ts,
raw=info
)
await self.callback(OPEN_INTEREST, oi, timestamp)
if 'index_price_e4' in info:
i = Index(
self.id,
self.exchange_symbol_to_std_symbol(info['symbol']),
Decimal(info['index_price_e4']) * Decimal('1e-4'),
ts,
raw=info
)
await self.callback(INDEX, i, timestamp)
if 'funding_rate_e6' in info:
f = Funding(
self.id,
self.exchange_symbol_to_std_symbol(info['symbol']),
None,
Decimal(info['funding_rate_e6']) * Decimal('1e-6'),
info['next_funding_time'].timestamp(),
ts,
predicted_rate=Decimal(info['predicted_funding_rate_e6']) * Decimal('1e-6'),
raw=info
)
await self.callback(FUNDING, f, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{"topic":"trade.BTCUSD",
"data":[
{
"timestamp":"2019-01-22T15:04:33.461Z",
"symbol":"BTCUSD",
"side":"Buy",
"size":980,
"price":3563.5,
"tick_direction":"PlusTick",
"trade_id":"9d229f26-09a8-42f8-aff3-0ba047b0449d",
"cross_seq":163261271}]}
"""
data = msg['data']
for trade in data:
if isinstance(trade['trade_time_ms'], str):
ts = int(trade['trade_time_ms'])
else:
ts = trade['trade_time_ms']
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(trade['symbol']),
BUY if trade['side'] == 'Buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
self.timestamp_normalize(ts),
id=trade['trade_id'],
raw=trade
)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['topic'].split('.')[-1])
update_type = msg['type']
data = msg['data']
delta = {BID: [], ASK: []}
if update_type == 'snapshot':
delta = None
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth)
# the USDT perpetual data is under the order_book key
if 'order_book' in data:
data = data['order_book']
for update in data:
side = BID if update['side'] == 'Buy' else ASK
self._l2_book[pair].book[side][Decimal(update['price'])] = Decimal(update['size'])
else:
for delete in data['delete']:
side = BID if delete['side'] == 'Buy' else ASK
price = Decimal(delete['price'])
delta[side].append((price, 0))
del self._l2_book[pair].book[side][price]
for utype in ('update', 'insert'):
for update in data[utype]:
side = BID if update['side'] == 'Buy' else ASK
price = Decimal(update['price'])
amount = Decimal(update['size'])
delta[side].append((price, amount))
self._l2_book[pair].book[side][price] = amount
# timestamp is in microseconds
ts = msg['timestamp_e6']
if isinstance(ts, str):
ts = int(ts)
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=ts / 1000000, raw=msg, delta=delta)
async def _order(self, msg: dict, timestamp: float):
"""
{
"topic": "order",
"action": "",
"data": [
{
"order_id": "xxxxxxxx-xxxx-xxxx-9a8f-4a973eb5c418",
"order_link_id": "",
"symbol": "BTCUSDT",
"side": "Buy",
"order_type": "Limit",
"price": 11000,
"qty": 0.001,
"leaves_qty": 0.001,
"last_exec_price": 0,
"cum_exec_qty": 0,
"cum_exec_value": 0,
"cum_exec_fee": 0,
"time_in_force": "GoodTillCancel",
"create_type": "CreateByUser",
"cancel_type": "UNKNOWN",
"order_status": "New",
"take_profit": 0,
"stop_loss": 0,
"trailing_stop": 0,
"reduce_only": false,
"close_on_trigger": false,
"create_time": "2020-08-12T21:18:40.780039678Z",
"update_time": "2020-08-12T21:18:40.787986415Z"
}
]
}
"""
order_status = {
'Created': SUBMITTING,
'Rejected': FAILED,
'New': OPEN,
'PartiallyFilled': PARTIAL,
'Filled': FILLED,
'Cancelled': CANCELLED,
'PendingCancel': CANCELLING
}
for i in range(len(msg['data'])):
data = msg['data'][i]
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
data["order_id"],
BUY if data["side"] == 'Buy' else SELL,
order_status[data["order_status"]],
LIMIT if data['order_type'] == 'Limit' else MARKET,
Decimal(data['price']),
Decimal(data['cum_exec_qty']),
Decimal(data['qty']) - Decimal(data['cum_exec_qty']),
self.timestamp_normalize(data.get('update_time') or data.get('O') or data.get('timestamp')),
raw=data,
)
await self.callback(ORDER_INFO, oi, timestamp)
async def _execution(self, msg: dict, timestamp: float):
'''
{
"topic": "execution",
"data": [
{
"symbol": "BTCUSD",
"side": "Buy",
"order_id": "xxxxxxxx-xxxx-xxxx-9a8f-4a973eb5c418",
"exec_id": "xxxxxxxx-xxxx-xxxx-8b66-c3d2fcd352f6",
"order_link_id": "",
"price": "8300",
"order_qty": 1,
"exec_type": "Trade",
"exec_qty": 1,
"exec_fee": "0.00000009",
"leaves_qty": 0,
"is_maker": false,
"trade_time": "2020-01-14T14:07:23.629Z" // trade time
}
]
}
'''
for entry in msg['data']:
symbol = self.exchange_symbol_to_std_symbol(entry['symbol'])
f = Fill(
self.id,
symbol,
BUY if entry['side'] == 'Buy' else SELL,
Decimal(entry['exec_qty']),
Decimal(entry['price']),
Decimal(entry['exec_fee']),
entry['exec_id'],
entry['order_id'],
None,
MAKER if entry['is_maker'] else TAKER,
entry['trade_time'].timestamp(),
raw=entry
)
await self.callback(FILLS, f, timestamp)
# async def _balances(self, msg: dict, timestamp: float):
# for i in range(len(msg['data'])):
# data = msg['data'][i]
# symbol = self.exchange_symbol_to_std_symbol(data['symbol'])
# await self.callback(BALANCES, feed=self.id, symbol=symbol, data=data, receipt_timestamp=timestamp)
async def authenticate(self, conn: AsyncConnection):
if any(self.is_authenticated_channel(self.exchange_channel_to_std(chan)) for chan in conn.subscription):
auth = self._auth(self.key_id, self.key_secret)
LOG.debug(f"{conn.uuid}: Sending authentication request with message {auth}")
await conn.write(auth)
def _auth(self, key_id: str, key_secret: str) -> str:
# https://bybit-exchange.github.io/docs/inverse/#t-websocketauthentication
expires = int((time.time() + 60)) * 1000
signature = str(hmac.new(bytes(key_secret, 'utf-8'), bytes(f'GET/realtime{expires}', 'utf-8'), digestmod='sha256').hexdigest())
return json.dumps({'op': 'auth', 'args': [key_id, expires, signature]})
class FMFW(Feed):
id = FMFW_id
websocket_endpoints = [
WebsocketEndpoint('wss://api.fmfw.io/api/3/ws/public')
]
rest_endpoints = [
RestEndpoint('https://api.fmfw.io',
routes=Routes('/api/3/public/symbol'))
]
valid_candle_intervals = {
'1m', '3m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '1M'
}
candle_interval_map = {
'1m': 'M1',
'3m': 'M3',
'5m': 'M5',
'15m': 'M15',
'30m': 'M30',
'1h': 'H1',
'4h': 'H4',
'1d': 'D1',
'1w': 'D7',
'1M': '1M'
}
websocket_channels = {
L2_BOOK: 'orderbook/full',
TRADES: 'trades',
TICKER: 'ticker/1s',
CANDLES: 'candles/'
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for sym, symbol in data.items():
s = Symbol(symbol['base_currency'], symbol['quote_currency'])
ret[s.normalized] = sym
info['tick_size'][s.normalized] = symbol['tick_size']
info['instrument_type'][s.normalized] = s.type
return ret, info
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
def __reset(self):
self._l2_book = {}
self.seq_no = {}
async def _book(self, msg: dict, ts: float):
if 'snapshot' in msg:
for pair, update in msg['snapshot'].items():
symbol = self.exchange_symbol_to_std_symbol(pair)
bids = {
Decimal(price): Decimal(size)
for price, size in update['b']
}
asks = {
Decimal(price): Decimal(size)
for price, size in update['a']
}
self._l2_book[symbol] = OrderBook(self.id,
pair,
max_depth=self.max_depth,
bids=bids,
asks=asks)
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
ts,
sequence_number=update['s'],
delta=None,
raw=msg,
timestamp=self.timestamp_normalize(
update['t']))
self.seq_no[symbol] = update['s']
else:
delta = {BID: [], ASK: []}
for pair, update in msg['update'].items():
symbol = self.exchange_symbol_to_std_symbol(pair)
if self.seq_no[symbol] + 1 != update['s']:
raise MissingSequenceNumber
self.seq_no[symbol] = update['s']
for side, key in ((BID, 'b'), (ASK, 'a')):
for price, size in update[key]:
price = Decimal(price)
size = Decimal(size)
delta[side].append((price, size))
if size == 0:
del self._l2_book[symbol].book[side][price]
else:
self._l2_book[symbol].book[side][price] = size
await self.book_callback(
L2_BOOK,
self._l2_book[symbol],
ts,
sequence_number=update['s'],
delta=delta,
raw=msg,
timestamp=self.timestamp_normalize(update['t']))
async def _trade(self, msg: dict, ts: float):
'''
{
'ch': 'trades',
'update': {
'BTCUSDT': [{
't': 1633803835228,
'i': 1633803835228,
'p': '54774.60',
'q': '0.00004',
's': 'buy'
}]
}
}
'''
for pair, update in msg['update'].items():
symbol = self.exchange_symbol_to_std_symbol(pair)
for trade in update:
t = Trade(self.id,
symbol,
BUY if trade['s'] == 'buy' else SELL,
Decimal(trade['q']),
Decimal(trade['p']),
self.timestamp_normalize(trade['t']),
id=str(trade['i']),
raw=msg)
await self.callback(TRADES, t, ts)
async def _ticker(self, msg: dict, ts: float):
'''
{
'ch': 'ticker/1s',
'data': {
'BTCUSDT': {
't': 1633804289795,
'a': '54813.56',
'A': '0.82000',
'b': '54810.31',
'B': '0.00660',
'o': '54517.48',
'c': '54829.88',
'h': '55493.92',
'l': '53685.61',
'v': '19025.22558',
'q': '1040244549.4048389',
'p': '312.40',
'P': '0.5730272198935094',
'L': 1417964345
}
}
}
'''
for sym, ticker in msg['data'].items():
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(sym),
Decimal(ticker['b']),
Decimal(ticker['a']),
self.timestamp_normalize(ticker['t']),
raw=msg)
await self.callback(TICKER, t, ts)
async def _candle(self, msg: dict, ts: float):
'''
{
'ch': 'candles/M1',
'update': {
'BTCUSDT': [{
't': 1633805940000,
'o': '54849.03',
'c': '54849.03',
'h': '54849.03',
'l': '54849.03',
'v': '0.00766',
'q': '420.1435698'
}]
}
}
'''
interval = msg['ch'].split("/")[-1]
for sym, updates in msg['update'].items():
symbol = self.exchange_symbol_to_std_symbol(sym)
for u in updates:
c = Candle(self.id,
symbol,
u['t'] / 1000,
u['t'] / 1000 + timedelta_str_to_sec(
self.normalize_candle_interval[interval]) - 0.1,
self.normalize_candle_interval[interval],
None,
Decimal(u['o']),
Decimal(u['c']),
Decimal(u['h']),
Decimal(u['l']),
Decimal(u['v']),
None,
self.timestamp_normalize(u['t']),
raw=msg)
await self.callback(CANDLES, c, ts)
async def message_handler(self, msg: str, conn, ts: float):
msg = json.loads(msg, parse_float=Decimal)
if 'result' in msg:
LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg)
elif msg['ch'] == 'orderbook/full':
await self._book(msg, ts)
elif msg['ch'] == 'trades':
await self._trade(msg, ts)
elif msg['ch'] == 'ticker/1s':
await self._ticker(msg, ts)
elif msg['ch'].startswith('candles/'):
await self._candle(msg, ts)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn):
self.__reset()
for chan in self.subscription:
await conn.write(
json.dumps({
"method":
"subscribe",
"params": {
"symbols": self.subscription[chan]
},
"ch":
chan if chan != 'candles/' else chan +
self.candle_interval_map[self.candle_interval],
"id":
1234
}))
class Bitstamp(Feed, BitstampRestMixin):
id = BITSTAMP
# API documentation: https://www.bitstamp.net/websocket/v2/
websocket_endpoints = [WebsocketEndpoint('wss://ws.bitstamp.net/', options={'compression': None})]
rest_endpoints = [RestEndpoint('https://www.bitstamp.net', routes=Routes('/api/v2/trading-pairs-info/', l2book='/api/v2/order_book/{}'))]
websocket_channels = {
L3_BOOK: 'detail_order_book',
L2_BOOK: 'diff_order_book',
TRADES: 'live_trades',
}
request_limit = 13
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1_000_000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
for d in data:
if d['trading'] != 'Enabled':
continue
base, quote = d['name'].split("/")
s = Symbol(base, quote)
symbol = d['url_symbol']
ret[s.normalized] = symbol
info['instrument_type'][s.normalized] = s.type
return ret, info
async def _process_l2_book(self, msg: dict, timestamp: float):
data = msg['data']
chan = msg['channel']
ts = int(data['microtimestamp'])
pair = self.exchange_symbol_to_std_symbol(chan.split('_')[-1])
delta = {BID: [], ASK: []}
if pair in self.last_update_id:
if data['timestamp'] < self.last_update_id[pair]:
return
else:
del self.last_update_id[pair]
for side in (BID, ASK):
for update in data[side + 's']:
price = Decimal(update[0])
size = Decimal(update[1])
if size == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
delta[side].append((price, size))
else:
self._l2_book[pair].book[side][price] = size
delta[side].append((price, size))
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(ts), delta=delta, raw=msg)
async def _process_l3_book(self, msg: dict, timestamp: float):
data = msg['data']
chan = msg['channel']
ts = int(data['microtimestamp'])
pair = self.exchange_symbol_to_std_symbol(chan.split('_')[-1])
book = OrderBook(self.id, pair, max_depth=self.max_depth)
for side in (BID, ASK):
for price, size, order_id in data[side + 's']:
price = Decimal(price)
size = Decimal(size)
if price in book.book[side]:
book.book[side][price][order_id] = size
else:
book.book[side][price] = {order_id: size}
self._l3_book[pair] = book
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=self.timestamp_normalize(ts), raw=msg)
async def _trades(self, msg: dict, timestamp: float):
"""
{'data':
{
'microtimestamp': '1562650233964229', // Event time (micros)
'amount': Decimal('0.014140160000000001'), // Quantity
'buy_order_id': 3709484695, // Buyer order ID
'sell_order_id': 3709484799, // Seller order ID
'amount_str': '0.01414016', // Quantity string
'price_str': '12700.00', // Price string
'timestamp': '1562650233', // Event time
'price': Decimal('12700.0'), // Price
'type': 1,
'id': 93215787
},
'event': 'trade',
'channel': 'live_trades_btcusd'
}
"""
data = msg['data']
chan = msg['channel']
pair = self.exchange_symbol_to_std_symbol(chan.split('_')[-1])
t = Trade(
self.id,
pair,
BUY if data['type'] == 0 else SELL,
Decimal(data['amount']),
Decimal(data['price']),
self.timestamp_normalize(int(data['microtimestamp'])),
id=str(data['id']),
raw=msg
)
await self.callback(TRADES, t, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'bts' in msg['event']:
if msg['event'] == 'bts:connection_established':
pass
elif msg['event'] == 'bts:subscription_succeeded':
pass
else:
LOG.warning("%s: Unexpected message %s", self.id, msg)
elif msg['event'] == 'trade':
await self._trades(msg, timestamp)
elif msg['event'] == 'data':
if msg['channel'].startswith('diff_order_book'):
await self._process_l2_book(msg, timestamp)
if msg['channel'].startswith('detail_order_book'):
await self._process_l3_book(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _snapshot(self, pairs: list, conn: AsyncConnection):
await asyncio.sleep(5)
urls = [self.rest_endpoints[0].route('l2book', self.sandbox).format(sym) for sym in pairs]
results = [await self.http_conn.read(url) for url in urls]
results = [json.loads(resp, parse_float=Decimal) for resp in results]
for r, pair in zip(results, pairs):
std_pair = self.exchange_symbol_to_std_symbol(pair) if pair else 'BTC-USD'
self.last_update_id[std_pair] = r['timestamp']
self._l2_book[std_pair] = OrderBook(self.id, std_pair, max_depth=self.max_depth, asks={Decimal(u[0]): Decimal(u[1]) for u in r['asks']}, bids={Decimal(u[0]): Decimal(u[1]) for u in r['bids']})
await self.book_callback(L2_BOOK, self._l2_book[std_pair], time.time(), timestamp=float(r['timestamp']), raw=r)
async def subscribe(self, conn: AsyncConnection):
snaps = []
self.last_update_id = {}
for chan in self.subscription:
for pair in self.subscription[chan]:
await conn.write(
json.dumps({
"event": "bts:subscribe",
"data": {
"channel": f"{chan}_{pair}"
}
}))
if 'diff_order_book' in chan:
snaps.append(pair)
await self._snapshot(snaps, conn)
class OKCoin(Feed):
id = OKCOIN
websocket_endpoints = [WebsocketEndpoint('wss://real.okcoin.com:8443/ws/v3')]
rest_endpoints = [RestEndpoint('https://www.okcoin.com', routes=Routes('/api/spot/v3/instruments'))]
websocket_channels = {
L2_BOOK: 'spot/depth_l2_tbt',
TRADES: 'spot/trade',
TICKER: 'spot/ticker',
}
@classmethod
def timestamp_normalize(cls, ts) -> float:
return ts.timestamp()
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for e in data:
s = Symbol(e['base_currency'], e['quote_currency'])
ret[s.normalized] = e['instrument_id']
info['tick_size'][s.normalized] = e['tick_size']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
for symbol in self.subscription[chan]:
request = {"op": "subscribe", "args": f"{chan}:{symbol}"}
await conn.write(json.dumps(request))
async def _ticker(self, msg: dict, timestamp: float):
"""
{
'table': 'spot/ticker',
'data': [
{
'instrument_id': 'BTC-USD',
'last': '3977.74',
'best_bid': '3977.08',
'best_ask': '3978.73',
'open_24h': '3978.21',
'high_24h': '3995.43',
'low_24h': '3961.02',
'base_volume_24h': '248.245',
'quote_volume_24h': '988112.225861',
'timestamp': '2019-03-22T22:26:34.019Z'
}
]
}
"""
for update in msg['data']:
pair = update['instrument_id']
update_timestamp = self.timestamp_normalize(update['timestamp'])
t = Ticker(self.id, pair, Decimal(update['best_bid']) if update['best_bid'] else Decimal(0), Decimal(update['best_ask']) if update['best_ask'] else Decimal(0), update_timestamp, raw=update)
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{'table': 'spot/trade', 'data': [{'instrument_id': 'BTC-USD', 'price': '3977.44', 'side': 'buy', 'size': '0.0096', 'timestamp': '2019-03-22T22:45:44.578Z', 'trade_id': '486519521'}]}
"""
for trade in msg['data']:
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(trade['instrument_id']),
BUY if trade['side'] == 'buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
self.timestamp_normalize(trade['timestamp']),
id=trade['trade_id'],
raw=trade
)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
if msg['action'] == 'partial':
# snapshot
for update in msg['data']:
pair = self.exchange_symbol_to_std_symbol(update['instrument_id'])
bids = {Decimal(price): Decimal(amount) for price, amount, *_ in update['bids']}
asks = {Decimal(price): Decimal(amount) for price, amount, *_ in update['asks']}
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, checksum_format='OKCOIN', bids=bids, asks=asks)
if self.checksum_validation and self._l2_book[pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(update['timestamp']), raw=msg, checksum=update['checksum'] & 0xFFFFFFFF)
else:
# update
for update in msg['data']:
delta = {BID: [], ASK: []}
pair = self.exchange_symbol_to_std_symbol(update['instrument_id'])
for side in ('bids', 'asks'):
s = BID if side == 'bids' else ASK
for price, amount, *_ in update[side]:
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
if price in self._l2_book[pair].book[s]:
delta[s].append((price, 0))
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, amount))
self._l2_book[pair].book[s][price] = amount
if self.checksum_validation and self._l2_book[pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(update['timestamp']), raw=msg, delta=delta, checksum=update['checksum'] & 0xFFFFFFFF)
async def message_handler(self, msg: str, conn, timestamp: float):
# DEFLATE compression, no header
msg = zlib.decompress(msg, -15)
msg = json.loads(msg, parse_float=Decimal)
if 'event' in msg:
if msg['event'] == 'error':
LOG.error("%s: Error: %s", self.id, msg)
elif msg['event'] == 'subscribe':
pass
else:
LOG.warning("%s: Unhandled event %s", self.id, msg)
elif 'table' in msg:
if 'ticker' in msg['table']:
await self._ticker(msg, timestamp)
elif 'trade' in msg['table']:
await self._trade(msg, timestamp)
elif 'depth_l2_tbt' in msg['table']:
await self._book(msg, timestamp)
elif 'spot/order' in msg['table']:
await self._order(msg, timestamp)
else:
LOG.warning("%s: Unhandled message %s", self.id, msg)
else:
LOG.warning("%s: Unhandled message %s", self.id, msg)
class EXX(Feed):
id = EXX_id
websocket_endpoints = [WebsocketEndpoint('wss://ws.exx.com/websocket')]
rest_endpoints = [
RestEndpoint('https://api.exx.com', routes=Routes('/data/v1/tickers'))
]
websocket_channels = {
L2_BOOK: 'ENTRUST_ADD',
TRADES: 'TRADE',
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
exchange = [key.upper() for key in data.keys()]
for sym in exchange:
b, q = sym.split("_")
s = Symbol(b, q)
ret[s.normalized] = sym
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
async def _book_update(self, msg: dict, timestamp: float):
"""
Snapshot:
[
[
'AE',
'1',
'BTC_USDT',
'1547941504',
{
'asks':[
[
'25000.00000000',
'0.02000000'
],
[
'19745.83000000',
'0.00200000'
],
[
'19698.96000000',
'0.00100000'
],
...
]
},
{
'bids':[
[
'3662.83040000',
'0.00100000'
],
[
'3662.77540000',
'0.01000000'
],
[
'3662.59900000',
'0.10300000'
],
...
]
}
]
]
Update:
['E', '1', '1547942636', 'BTC_USDT', 'ASK', '3674.91740000', '0.02600000']
"""
delta = {BID: [], ASK: []}
if msg[0] == 'AE':
# snapshot
delta = None
pair = self.exchange_symbol_to_std_symbol(msg[2])
ts = msg[3]
asks = msg[4]['asks'] if 'asks' in msg[4] else msg[5]['asks']
bids = msg[5]['bids'] if 'bids' in msg[5] else msg[4]['bids']
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for price, amount in bids
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for price, amount in asks
}
else:
# Update
ts = msg[2]
pair = self.exchange_symbol_to_std_symbol(msg[3])
side = ASK if msg[4] == 'ASK' else BID
price = Decimal(msg[5])
amount = Decimal(msg[6])
if amount == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair][side].book[price]
delta[side].append((price, 0))
else:
self._l2_book[pair].book[side][price] = amount
delta[side].append((price, amount))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=ts,
raw=msg,
delta=delta)
async def _trade(self, msg: dict, timestamp: float):
"""
Trade message
['T', '1', '1547947390', 'BTC_USDT', 'bid', '3683.74440000', '0.082', '33732290']
"""
pair = self.exchange_symbol_to_std_symbol(msg[3])
t = Trade(self.id,
pair,
BUY if msg[4] == 'bid' else SELL,
Decimal(msg[6]),
Decimal(msg[5]),
float(msg[2]),
id=msg[7],
raw=msg)
await self.callback(TRADES, t, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if isinstance(msg[0], list):
msg = msg[0]
if msg[0] == 'E' or msg[0] == 'AE':
await self._book_update(msg, timestamp)
elif msg[0] == 'T':
await self._trade(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
for pair in self.subscription[chan]:
await conn.write(
json.dumps({
"dataType": f"1_{chan}_{pair}",
"dataSize": 50,
"action": "ADD"
}))
class BinanceDelivery(Binance, BinanceDeliveryRestMixin):
id = BINANCE_DELIVERY
websocket_endpoints = [
WebsocketEndpoint('wss://dstream.binance.com',
options={'compression': None})
]
rest_endpoints = [
RestEndpoint('https://dapi.binance.com',
routes=Routes('/dapi/v1/exchangeInfo',
l2book='/dapi/v1/depth?symbol={}&limit={}',
authentication='/dapi/v1/listenKey'))
]
valid_depths = [5, 10, 20, 50, 100, 500, 1000]
valid_depth_intervals = {'100ms', '250ms', '500ms'}
websocket_channels = {
**Binance.websocket_channels, FUNDING: 'markPrice',
OPEN_INTEREST: 'open_interest',
LIQUIDATIONS: 'forceOrder',
POSITIONS: POSITIONS
}
def _check_update_id(self, pair: str, msg: dict) -> bool:
if self._l2_book[pair].delta is None and msg[
'u'] < self.last_update_id[pair]:
return True
elif msg['U'] <= self.last_update_id[pair] <= msg['u']:
self.last_update_id[pair] = msg['u']
return False
elif self.last_update_id[pair] == msg['pu']:
self.last_update_id[pair] = msg['u']
return False
else:
self._reset()
LOG.warning("%s: Missing book update detected, resetting book",
self.id)
return True
async def _account_update(self, msg: dict, timestamp: float):
"""
{
"e": "ACCOUNT_UPDATE", // Event Type
"E": 1564745798939, // Event Time
"T": 1564745798938 , // Transaction
"i": "SfsR", // Account Alias
"a": // Update Data
{
"m":"ORDER", // Event reason type
"B":[ // Balances
{
"a":"BTC", // Asset
"wb":"122624.12345678", // Wallet Balance
"cw":"100.12345678" // Cross Wallet Balance
},
{
"a":"ETH",
"wb":"1.00000000",
"cw":"0.00000000"
}
],
"P":[
{
"s":"BTCUSD_200925", // Symbol
"pa":"0", // Position Amount
"ep":"0.0", // Entry Price
"cr":"200", // (Pre-fee) Accumulated Realized
"up":"0", // Unrealized PnL
"mt":"isolated", // Margin Type
"iw":"0.00000000", // Isolated Wallet (if isolated position)
"ps":"BOTH" // Position Side
},
{
"s":"BTCUSD_200925",
"pa":"20",
"ep":"6563.6",
"cr":"0",
"up":"2850.21200000",
"mt":"isolated",
"iw":"13200.70726908",
"ps":"LONG"
},
{
"s":"BTCUSD_200925",
"pa":"-10",
"ep":"6563.8",
"cr":"-45.04000000",
"up":"-1423.15600000",
"mt":"isolated",
"iw":"6570.42511771",
"ps":"SHORT"
}
]
}
}
"""
for balance in msg['a']['B']:
b = Balance(self.id,
balance['a'],
Decimal(balance['wb']),
None,
raw=msg)
await self.callback(BALANCES, b, timestamp)
for position in msg['a']['P']:
p = Position(self.id,
self.exchange_symbol_to_std_symbol(position['s']),
Decimal(position['pa']),
Decimal(position['ep']),
position['ps'].lower(),
Decimal(position['up']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(POSITIONS, p, timestamp)
async def _order_update(self, msg: dict, timestamp: float):
"""
{
"e":"ORDER_TRADE_UPDATE", // Event Type
"E":1591274595442, // Event Time
"T":1591274595453, // Transaction Time
"i":"SfsR", // Account Alias
"o":
{
"s":"BTCUSD_200925", // Symbol
"c":"TEST", // Client Order Id
// special client order id:
// starts with "autoclose-": liquidation order
// "adl_autoclose": ADL auto close order
"S":"SELL", // Side
"o":"TRAILING_STOP_MARKET", // Order Type
"f":"GTC", // Time in Force
"q":"2", // Original Quantity
"p":"0", // Original Price
"ap":"0", // Average Price
"sp":"9103.1", // Stop Price. Please ignore with TRAILING_STOP_MARKET order
"x":"NEW", // Execution Type
"X":"NEW", // Order Status
"i":8888888, // Order Id
"l":"0", // Order Last Filled Quantity
"z":"0", // Order Filled Accumulated Quantity
"L":"0", // Last Filled Price
"ma": "BTC", // Margin Asset
"N":"BTC", // Commission Asset of the trade, will not push if no commission
"n":"0", // Commission of the trade, will not push if no commission
"T":1591274595442, // Order Trade Time
"t":0, // Trade Id
"rp": "0", // Realized Profit of the trade
"b":"0", // Bid quantity of base asset
"a":"0", // Ask quantity of base asset
"m":false, // Is this trade the maker side?
"R":false, // Is this reduce only
"wt":"CONTRACT_PRICE", // Stop Price Working Type
"ot":"TRAILING_STOP_MARKET",// Original Order Type
"ps":"LONG", // Position Side
"cp":false, // If Close-All, pushed with conditional order
"AP":"9476.8", // Activation Price, only puhed with TRAILING_STOP_MARKET order
"cr":"5.0", // Callback Rate, only puhed with TRAILING_STOP_MARKET order
"pP": false // If conditional order trigger is protected
}
}
"""
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(msg['o']['s']),
str(msg['o']['i']),
BUY if msg['o']['S'].lower() == 'buy' else SELL,
msg['o']['x'],
LIMIT if msg['o']['o'].lower() == 'limit' else
MARKET if msg['o']['o'].lower() == 'market' else None,
Decimal(msg['o']['ap']) if
not Decimal.is_zero(Decimal(msg['o']['ap'])) else None,
Decimal(msg['o']['q']),
Decimal(msg['o']['q']) - Decimal(msg['o']['z']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Handle account updates from User Data Stream
if self.requires_authentication:
msg_type = msg.get('e')
if msg_type == 'ACCOUNT_UPDATE':
await self._account_update(msg, timestamp)
elif msg_type == 'ORDER_TRADE_UPDATE':
await self._order_update(msg, timestamp)
return
# Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>}
# streamName is of format <symbol>@<channel>
pair, _ = msg['stream'].split('@', 1)
msg = msg['data']
pair = pair.upper()
msg_type = msg.get('e')
if msg_type == 'bookTicker':
await self._ticker(msg, timestamp)
elif msg_type == 'depthUpdate':
await self._book(msg, pair, timestamp)
elif msg_type == 'aggTrade':
await self._trade(msg, timestamp)
elif msg_type == 'forceOrder':
await self._liquidations(msg, timestamp)
elif msg_type == 'markPriceUpdate':
await self._funding(msg, timestamp)
elif msg_type == 'kline':
await self._candle(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class Bitmex(Feed, BitmexRestMixin):
id = BITMEX
websocket_endpoints = [WebsocketEndpoint('wss://www.bitmex.com/realtime', sandbox='wss://testnet.bitmex.com/realtime', options={'compression': None})]
rest_endpoints = [RestEndpoint('https://www.bitmex.com', routes=Routes('/api/v1/instrument/active'))]
websocket_channels = {
L2_BOOK: 'orderBookL2',
TRADES: 'trade',
TICKER: 'quote',
FUNDING: 'funding',
ORDER_INFO: 'order',
OPEN_INTEREST: 'instrument',
LIQUIDATIONS: 'liquidation'
}
request_limit = 0.5
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
base = entry['rootSymbol'].replace("XBT", "BTC")
quote = entry['quoteCurrency'].replace("XBT", "BTC")
stype = PERPETUAL
if entry['expiry']:
stype = FUTURES
s = Symbol(base, quote, type=stype, expiry_date=entry['expiry'])
ret[s.normalized] = entry['symbol']
info['tick_size'][s.normalized] = entry['tickSize']
info['instrument_type'][s.normalized] = stype
return ret, info
def _reset(self):
self.partial_received = defaultdict(bool)
self.order_id = {}
self.open_orders = {}
for pair in self.normalized_symbols:
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth)
self.order_id[pair] = defaultdict(dict)
@staticmethod
def normalize_order_status(status):
status_map = {
'New': OPEN,
'Filled': FILLED,
'Canceled': CANCELLED,
}
return status_map[status]
def init_order_info(self, o):
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(o['symbol']),
o['orderID'],
BUY if o['side'] == 'Buy' else SELL,
self.normalize_order_status(o['ordStatus']),
LIMIT if o['ordType'].lower() == 'limit' else MARKET if o['ordType'].lower() == 'market' else None,
Decimal(o['avgPx']) if o['avgPx'] else Decimal(o['price']),
Decimal(o['orderQty']),
Decimal(o['leavesQty']),
self.timestamp_normalize(o['timestamp']),
raw=str(o), # Need to convert to string to avoid json serialization error when updating order
)
return oi
async def _order(self, msg: dict, timestamp: float):
"""
order msg example
{
"table": "order",
"action": "partial",
"keys": [
"orderID"
],
"types": {
"orderID": "guid",
"clOrdID": "string",
"clOrdLinkID": "symbol",
"account": "long",
"symbol": "symbol",
"side": "symbol",
"simpleOrderQty": "float",
"orderQty": "long",
"price": "float",
"displayQty": "long",
"stopPx": "float",
"pegOffsetValue": "float",
"pegPriceType": "symbol",
"currency": "symbol",
"settlCurrency": "symbol",
"ordType": "symbol",
"timeInForce": "symbol",
"execInst": "symbol",
"contingencyType": "symbol",
"exDestination": "symbol",
"ordStatus": "symbol",
"triggered": "symbol",
"workingIndicator": "boolean",
"ordRejReason": "symbol",
"simpleLeavesQty": "float",
"leavesQty": "long",
"simpleCumQty": "float",
"cumQty": "long",
"avgPx": "float",
"multiLegReportingType": "symbol",
"text": "string",
"transactTime": "timestamp",
"timestamp": "timestamp"
},
"foreignKeys": {
"symbol": "instrument",
"side": "side",
"ordStatus": "ordStatus"
},
"attributes": {
"orderID": "grouped",
"account": "grouped",
"ordStatus": "grouped",
"workingIndicator": "grouped"
},
"filter": {
"account": 1600000,
"symbol": "ETHUSDTH22"
},
"data": [
{
"orderID": "360fad5a-49e3-4187-ad04-8fac82b8a95f",
"clOrdID": "",
"clOrdLinkID": "",
"account": 1600000,
"symbol": "ETHUSDTH22",
"side": "Buy",
"simpleOrderQty": null,
"orderQty": 1000,
"price": 2000,
"displayQty": null,
"stopPx": null,
"pegOffsetValue": null,
"pegPriceType": "",
"currency": "USDT",
"settlCurrency": "USDt",
"ordType": "Limit",
"timeInForce": "GoodTillCancel",
"execInst": "",
"contingencyType": "",
"exDestination": "XBME",
"ordStatus": "New",
"triggered": "",
"workingIndicator": true,
"ordRejReason": "",
"simpleLeavesQty": null,
"leavesQty": 1000,
"simpleCumQty": null,
"cumQty": 0,
"avgPx": null,
"multiLegReportingType": "SingleSecurity",
"text": "Submitted via API.",
"transactTime": "2022-02-13T00:15:02.570000Z",
"timestamp": "2022-02-13T00:15:02.570000Z"
},
{
"orderID": "74d2ad0a-49f1-44dc-820f-5f0cfd64c1a3",
"clOrdID": "",
"clOrdLinkID": "",
"account": 1600000,
"symbol": "ETHUSDTH22",
"side": "Buy",
"simpleOrderQty": null,
"orderQty": 1000,
"price": 2000,
"displayQty": null,
"stopPx": null,
"pegOffsetValue": null,
"pegPriceType": "",
"currency": "USDT",
"settlCurrency": "USDt",
"ordType": "Limit",
"timeInForce": "GoodTillCancel",
"execInst": "",
"contingencyType": "",
"exDestination": "XBME",
"ordStatus": "New",
"triggered": "",
"workingIndicator": true,
"ordRejReason": "",
"simpleLeavesQty": null,
"leavesQty": 1000,
"simpleCumQty": null,
"cumQty": 0,
"avgPx": null,
"multiLegReportingType": "SingleSecurity",
"text": "Submitted via API.",
"transactTime": "2022-02-13T00:17:13.796000Z",
"timestamp": "2022-02-13T00:17:13.796000Z"
}
]
}
{
"table": "order",
"action": "insert",
"data": [
{
"orderID": "0c4e4a8e-b234-495f-8b94-c4766786c4a5",
"clOrdID": "",
"clOrdLinkID": "",
"account": 1600000,
"symbol": "ETHUSDTH22",
"side": "Buy",
"simpleOrderQty": null,
"orderQty": 1000,
"price": 2000,
"displayQty": null,
"stopPx": null,
"pegOffsetValue": null,
"pegPriceType": "",
"currency": "USDT",
"settlCurrency": "USDt",
"ordType": "Limit",
"timeInForce": "GoodTillCancel",
"execInst": "",
"contingencyType": "",
"exDestination": "XBME",
"ordStatus": "New",
"triggered": "",
"workingIndicator": true,
"ordRejReason": "",
"simpleLeavesQty": null,
"leavesQty": 1000,
"simpleCumQty": null,
"cumQty": 0,
"avgPx": null,
"multiLegReportingType": "SingleSecurity",
"text": "Submitted via API.",
"transactTime": "2022-02-13T00:21:50.268000Z",
"timestamp": "2022-02-13T00:21:50.268000Z"
}
]
}
{
"table": "order",
"action": "update",
"data": [
{
"orderID": "360fa95a-49e3-4187-ad04-8fac82b8a95f",
"ordStatus": "Canceled",
"workingIndicator": false,
"leavesQty": 0,
"text": "Canceled: Cancel from www.bitmex.com\nSubmitted via API.",
"timestamp": "2022-02-13T08:16:36.446000Z",
"clOrdID": "",
"account": 1600000,
"symbol": "ETHUSDTH22"
}
]
}
"""
if msg['action'] == 'partial':
# Initial snapshot of open orders
self.open_orders = {}
for o in msg['data']:
oi = self.init_order_info(o)
self.open_orders[oi.id] = oi
elif msg['action'] == 'insert':
for o in msg['data']:
oi = self.init_order_info(o)
self.open_orders[oi.id] = oi
await self.callback(ORDER_INFO, oi, timestamp)
elif msg['action'] == 'update':
for o in msg['data']:
oi = self.open_orders.get(o['orderID'])
if oi:
info = oi.to_dict()
if 'ordStatus' in o:
info['status'] = self.normalize_order_status(o['ordStatus'])
if 'leaveQty' in o:
info['remaining'] = Decimal(o['leavesQty'])
if 'avgPx' in o:
info['price'] = Decimal(o['avgPx'])
info['raw'] = str(o) # Not sure if this is needed
new_oi = OrderInfo(**info)
if new_oi.status in (FILLED, CANCELLED):
self.open_orders.pop(new_oi.id)
else:
self.open_orders[new_oi.id] = oi
await self.callback(ORDER_INFO, new_oi, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def _trade(self, msg: dict, timestamp: float):
"""
trade msg example
{
'timestamp': '2018-05-19T12:25:26.632Z',
'symbol': 'XBTUSD',
'side': 'Buy',
'size': 40,
'price': 8335,
'tickDirection': 'PlusTick',
'trdMatchID': '5f4ecd49-f87f-41c0-06e3-4a9405b9cdde',
'grossValue': 479920,
'homeNotional': Decimal('0.0047992'),
'foreignNotional': 40
}
"""
for data in msg['data']:
ts = self.timestamp_normalize(data['timestamp'])
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
BUY if data['side'] == 'Buy' else SELL,
Decimal(data['size']),
Decimal(data['price']),
ts,
id=data['trdMatchID'],
raw=data
)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
"""
the Full bitmex book
"""
# PERF perf_start(self.id, 'book_msg')
if not msg['data']:
# see https://github.com/bmoscon/cryptofeed/issues/688
# msg['data'] can be an empty list
return
delta = None
# if we reset the book, force a full update
pair = self.exchange_symbol_to_std_symbol(msg['data'][0]['symbol'])
if not self.partial_received[pair]:
# per bitmex documentation messages received before partial
# should be discarded
if msg['action'] != 'partial':
return
self.partial_received[pair] = True
if msg['action'] == 'partial':
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
price = Decimal(data['price'])
size = Decimal(data['size'])
order_id = data['id']
self._l2_book[pair].book[side][price] = size
self.order_id[pair][side][order_id] = price
elif msg['action'] == 'insert':
delta = {BID: [], ASK: []}
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
price = Decimal(data['price'])
size = Decimal(data['size'])
order_id = data['id']
self._l2_book[pair].book[side][price] = size
self.order_id[pair][side][order_id] = price
delta[side].append((price, size))
elif msg['action'] == 'update':
delta = {BID: [], ASK: []}
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
update_size = Decimal(data['size'])
order_id = data['id']
price = self.order_id[pair][side][order_id]
self._l2_book[pair].book[side][price] = update_size
self.order_id[pair][side][order_id] = price
delta[side].append((price, update_size))
elif msg['action'] == 'delete':
delta = {BID: [], ASK: []}
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
order_id = data['id']
delete_price = self.order_id[pair][side][order_id]
del self.order_id[pair][side][order_id]
del self._l2_book[pair].book[side][delete_price]
delta[side].append((delete_price, 0))
else:
LOG.warning("%s: Unexpected l2 Book message %s", self.id, msg)
return
# PERF perf_end(self.id, 'book_msg')
# PERF perf_log(self.id, 'book_msg')
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg, delta=delta)
async def _ticker(self, msg: dict, timestamp: float):
for data in msg['data']:
t = Ticker(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
Decimal(data['bidPrice']),
Decimal(data['askPrice']),
self.timestamp_normalize(data['timestamp']),
raw=data
)
await self.callback(TICKER, t, timestamp)
async def _funding(self, msg: dict, timestamp: float):
"""
{'table': 'funding',
'action': 'partial',
'keys': ['timestamp', 'symbol'],
'types': {
'timestamp': 'timestamp',
'symbol': 'symbol',
'fundingInterval': 'timespan',
'fundingRate': 'float',
'fundingRateDaily': 'float'
},
'foreignKeys': {
'symbol': 'instrument'
},
'attributes': {
'timestamp': 'sorted',
'symbol': 'grouped'
},
'filter': {'symbol': 'XBTUSD'},
'data': [{
'timestamp': '2018-08-21T20:00:00.000Z',
'symbol': 'XBTUSD',
'fundingInterval': '2000-01-01T08:00:00.000Z',
'fundingRate': Decimal('-0.000561'),
'fundingRateDaily': Decimal('-0.001683')
}]
}
"""
for data in msg['data']:
ts = self.timestamp_normalize(data['timestamp'])
interval = data['fundingInterval']
f = Funding(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
None,
data['fundingRate'],
self.timestamp_normalize(data['timestamp'] + timedelta(hours=interval.hour)),
ts,
raw=data
)
await self.callback(FUNDING, f, timestamp)
async def _instrument(self, msg: dict, timestamp: float):
"""
Example instrument data
{
'table':'instrument',
'action':'partial',
'keys':[
'symbol'
],
'types':{
'symbol':'symbol',
'rootSymbol':'symbol',
'state':'symbol',
'typ':'symbol',
'listing':'timestamp',
'front':'timestamp',
'expiry':'timestamp',
'settle':'timestamp',
'relistInterval':'timespan',
'inverseLeg':'symbol',
'sellLeg':'symbol',
'buyLeg':'symbol',
'optionStrikePcnt':'float',
'optionStrikeRound':'float',
'optionStrikePrice':'float',
'optionMultiplier':'float',
'positionCurrency':'symbol',
'underlying':'symbol',
'quoteCurrency':'symbol',
'underlyingSymbol':'symbol',
'reference':'symbol',
'referenceSymbol':'symbol',
'calcInterval':'timespan',
'publishInterval':'timespan',
'publishTime':'timespan',
'maxOrderQty':'long',
'maxPrice':'float',
'lotSize':'long',
'tickSize':'float',
'multiplier':'long',
'settlCurrency':'symbol',
'underlyingToPositionMultiplier':'long',
'underlyingToSettleMultiplier':'long',
'quoteToSettleMultiplier':'long',
'isQuanto':'boolean',
'isInverse':'boolean',
'initMargin':'float',
'maintMargin':'float',
'riskLimit':'long',
'riskStep':'long',
'limit':'float',
'capped':'boolean',
'taxed':'boolean',
'deleverage':'boolean',
'makerFee':'float',
'takerFee':'float',
'settlementFee':'float',
'insuranceFee':'float',
'fundingBaseSymbol':'symbol',
'fundingQuoteSymbol':'symbol',
'fundingPremiumSymbol':'symbol',
'fundingTimestamp':'timestamp',
'fundingInterval':'timespan',
'fundingRate':'float',
'indicativeFundingRate':'float',
'rebalanceTimestamp':'timestamp',
'rebalanceInterval':'timespan',
'openingTimestamp':'timestamp',
'closingTimestamp':'timestamp',
'sessionInterval':'timespan',
'prevClosePrice':'float',
'limitDownPrice':'float',
'limitUpPrice':'float',
'bankruptLimitDownPrice':'float',
'bankruptLimitUpPrice':'float',
'prevTotalVolume':'long',
'totalVolume':'long',
'volume':'long',
'volume24h':'long',
'prevTotalTurnover':'long',
'totalTurnover':'long',
'turnover':'long',
'turnover24h':'long',
'homeNotional24h':'float',
'foreignNotional24h':'float',
'prevPrice24h':'float',
'vwap':'float',
'highPrice':'float',
'lowPrice':'float',
'lastPrice':'float',
'lastPriceProtected':'float',
'lastTickDirection':'symbol',
'lastChangePcnt':'float',
'bidPrice':'float',
'midPrice':'float',
'askPrice':'float',
'impactBidPrice':'float',
'impactMidPrice':'float',
'impactAskPrice':'float',
'hasLiquidity':'boolean',
'openInterest':'long',
'openValue':'long',
'fairMethod':'symbol',
'fairBasisRate':'float',
'fairBasis':'float',
'fairPrice':'float',
'markMethod':'symbol',
'markPrice':'float',
'indicativeTaxRate':'float',
'indicativeSettlePrice':'float',
'optionUnderlyingPrice':'float',
'settledPrice':'float',
'timestamp':'timestamp'
},
'foreignKeys':{
'inverseLeg':'instrument',
'sellLeg':'instrument',
'buyLeg':'instrument'
},
'attributes':{
'symbol':'unique'
},
'filter':{
'symbol':'XBTUSD'
},
'data':[
{
'symbol':'XBTUSD',
'rootSymbol':'XBT',
'state':'Open',
'typ':'FFWCSX',
'listing':'2016-05-13T12:00:00.000Z',
'front':'2016-05-13T12:00:00.000Z',
'expiry':None,
'settle':None,
'relistInterval':None,
'inverseLeg':'',
'sellLeg':'',
'buyLeg':'',
'optionStrikePcnt':None,
'optionStrikeRound':None,
'optionStrikePrice':None,
'optionMultiplier':None,
'positionCurrency':'USD',
'underlying':'XBT',
'quoteCurrency':'USD',
'underlyingSymbol':'XBT=',
'reference':'BMEX',
'referenceSymbol':'.BXBT',
'calcInterval':None,
'publishInterval':None,
'publishTime':None,
'maxOrderQty':10000000,
'maxPrice':1000000,
'lotSize':1,
'tickSize':Decimal( '0.5' ),
'multiplier':-100000000,
'settlCurrency':'XBt',
'underlyingToPositionMultiplier':None,
'underlyingToSettleMultiplier':-100000000,
'quoteToSettleMultiplier':None,
'isQuanto':False,
'isInverse':True,
'initMargin':Decimal( '0.01' ),
'maintMargin':Decimal( '0.005' ),
'riskLimit':20000000000,
'riskStep':10000000000,
'limit':None,
'capped':False,
'taxed':True,
'deleverage':True,
'makerFee':Decimal( '-0.00025' ),
'takerFee':Decimal( '0.00075' ),
'settlementFee':0,
'insuranceFee':0,
'fundingBaseSymbol':'.XBTBON8H',
'fundingQuoteSymbol':'.USDBON8H',
'fundingPremiumSymbol':'.XBTUSDPI8H',
'fundingTimestamp':'2020-02-02T04:00:00.000Z',
'fundingInterval':'2000-01-01T08:00:00.000Z',
'fundingRate':Decimal( '0.000106' ),
'indicativeFundingRate':Decimal( '0.0001' ),
'rebalanceTimestamp':None,
'rebalanceInterval':None,
'openingTimestamp':'2020-02-02T00:00:00.000Z',
'closingTimestamp':'2020-02-02T01:00:00.000Z',
'sessionInterval':'2000-01-01T01:00:00.000Z',
'prevClosePrice':Decimal( '9340.63' ),
'limitDownPrice':None,
'limitUpPrice':None,
'bankruptLimitDownPrice':None,
'bankruptLimitUpPrice':None,
'prevTotalVolume':1999389257669,
'totalVolume':1999420432348,
'volume':31174679,
'volume24h':1605909209,
'prevTotalTurnover':27967114248663460,
'totalTurnover':27967447182062520,
'turnover':332933399058,
'turnover24h':17126993087717,
'homeNotional24h':Decimal( '171269.9308771703' ),
'foreignNotional24h':1605909209,
'prevPrice24h':9348,
'vwap':Decimal( '9377.3443' ),
'highPrice':9464,
'lowPrice':Decimal( '9287.5' ),
'lastPrice':9352,
'lastPriceProtected':9352,
'lastTickDirection':'ZeroMinusTick',
'lastChangePcnt':Decimal( '0.0004' ),
'bidPrice':9352,
'midPrice':Decimal( '9352.25' ),
'askPrice':Decimal( '9352.5' ),
'impactBidPrice':Decimal( '9351.9125' ),
'impactMidPrice':Decimal( '9352.25' ),
'impactAskPrice':Decimal( '9352.7871' ),
'hasLiquidity':True,
'openInterest':983043322,
'openValue':10518563545400,
'fairMethod':'FundingRate',
'fairBasisRate':Decimal( '0.11607' ),
'fairBasis':Decimal( '0.43' ),
'fairPrice':Decimal( '9345.36' ),
'markMethod':'FairPrice',
'markPrice':Decimal( '9345.36' ),
'indicativeTaxRate':0,
'indicativeSettlePrice':Decimal( '9344.93' ),
'optionUnderlyingPrice':None,
'settledPrice':None,
'timestamp':'2020-02-02T00:30:43.772Z'
}
]
}
"""
for data in msg['data']:
if 'openInterest' in data:
ts = self.timestamp_normalize(data['timestamp'])
oi = OpenInterest(self.id, self.exchange_symbol_to_std_symbol(data['symbol']), Decimal(data['openInterest']), ts, raw=data)
await self.callback(OPEN_INTEREST, oi, timestamp)
async def _liquidation(self, msg: dict, timestamp: float):
"""
liquidation msg example
{
'orderID': '9513c849-ca0d-4e11-8190-9d221972288c',
'symbol': 'XBTUSD',
'side': 'Buy',
'price': 6833.5,
'leavesQty': 2020
}
"""
if msg['action'] == 'insert':
for data in msg['data']:
liq = Liquidation(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
BUY if data['side'] == 'Buy' else SELL,
Decimal(data['leavesQty']),
Decimal(data['price']),
data['orderID'],
UNFILLED,
None,
raw=data
)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'table' in msg:
if msg['table'] == 'trade':
await self._trade(msg, timestamp)
elif msg['table'] == 'order':
await self._order(msg, timestamp)
elif msg['table'] == 'orderBookL2':
await self._book(msg, timestamp)
elif msg['table'] == 'funding':
await self._funding(msg, timestamp)
elif msg['table'] == 'instrument':
await self._instrument(msg, timestamp)
elif msg['table'] == 'quote':
await self._ticker(msg, timestamp)
elif msg['table'] == 'liquidation':
await self._liquidation(msg, timestamp)
else:
LOG.warning("%s: Unhandled table=%r in %r", conn.uuid, msg['table'], msg)
elif 'info' in msg:
LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg)
elif 'subscribe' in msg:
if not msg['success']:
LOG.error("%s: Subscribe failure: %s", conn.uuid, msg)
elif 'error' in msg:
LOG.error("%s: Error message from exchange: %s", conn.uuid, msg)
elif 'request' in msg:
if msg['success']:
LOG.debug("%s: Success %s", conn.uuid, msg['request'].get('op'))
else:
LOG.warning("%s: Failure %s", conn.uuid, msg['request'])
else:
LOG.warning("%s: Unexpected message from exchange: %s", conn.uuid, msg)
async def subscribe(self, conn: AsyncConnection):
self._reset()
await self._authenticate(conn)
chans = []
for chan in self.subscription:
for pair in self.subscription[chan]:
chans.append(f"{chan}:{pair}")
for i in range(0, len(chans), 10):
await conn.write(json.dumps({"op": "subscribe",
"args": chans[i:i + 10]}))
async def _authenticate(self, conn: AsyncConnection):
"""Send API Key with signed message."""
# Docs: https://www.bitmex.com/app/apiKeys
# https://github.com/BitMEX/sample-market-maker/blob/master/test/websocket-apikey-auth-test.py
if self.key_id and self.key_secret:
LOG.info('%s: Authenticate with signature', conn.uuid)
expires = int(time.time()) + 365 * 24 * 3600 # One year
msg = f'GET/realtime{expires}'.encode('utf-8')
signature = hmac.new(self.key_secret.encode('utf-8'), msg, digestmod=hashlib.sha256).hexdigest()
await conn.write(json.dumps({'op': 'authKeyExpires', 'args': [self.key_id, expires, signature]}))
class Probit(Feed):
id = PROBIT
websocket_endpoints = [
WebsocketEndpoint('wss://api.probit.com/api/exchange/v1/ws')
]
rest_endpoints = [
RestEndpoint('https://api.probit.com',
routes=Routes('/api/exchange/v1/market'))
]
websocket_channels = {
L2_BOOK: 'order_books',
TRADES: 'recent_trades',
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
# doc: https://docs-en.probit.com/reference-link/market
for entry in data['data']:
if entry['closed']:
continue
s = Symbol(entry['base_currency_id'], entry['quote_currency_id'])
ret[s.normalized] = entry['id']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
async def _trades(self, msg: dict, timestamp: float):
'''
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok","lag":0,
"recent_trades":[
{
"id":"ETH-BTC:4429182",
"price":"0.028229",
"quantity":"3.117",
"time":"2020-11-01T03:59:06.277Z",
"side":"buy","tick_direction":"down"
},{
"id":"ETH-BTC:4429183",
"price":"0.028227",
"quantity":"1.793",
"time":"2020-11-01T03:59:14.528Z",
"side":"buy",
"tick_direction":"down"
}
],"reset":true
}
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok","lag":0,
"recent_trades":[
{
"id":"ETH-BTC:4429282",
"price":"0.028235",
"quantity":"2.203",
"time":"2020-11-01T04:22:15.117Z",
"side":"buy",
"tick_direction":"down"
}
]
}
'''
pair = self.exchange_symbol_to_std_symbol(msg['market_id'])
for update in msg['recent_trades']:
t = Trade(self.id,
pair,
BUY if update['side'] == 'buy' else SELL,
Decimal(update['quantity']),
Decimal(update['price']),
self.timestamp_normalize(update['time']),
id=update['id'],
raw=update)
await self.callback(TRADES, t, timestamp)
async def _l2_update(self, msg: dict, timestamp: float):
'''
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok",
"lag":0,
"order_books":[
{
"side":"buy",
"price":"0.0165",
"quantity":"0.47"
},{
"side":"buy",
"price":"0",
"quantity":"14656.177"
},{
"side":"sell",
"price":"6400",
"quantity":"0.001"
}],
"reset":true
}
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok",
"lag":0,
"order_books":[
{
"side":"buy",
"price":"0.0281",
"quantity":"48.541"
},{
"side":"sell",
"price":"0.0283",
"quantity":"0"
}]
}
'''
pair = self.exchange_symbol_to_std_symbol(msg['market_id'])
is_snapshot = msg.get('reset', False)
if is_snapshot:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
for entry in msg["order_books"]:
price = Decimal(entry['price'])
quantity = Decimal(entry['quantity'])
side = BID if entry['side'] == "buy" else ASK
self._l2_book[pair].book[side][price] = quantity
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg)
else:
delta = {BID: [], ASK: []}
for entry in msg["order_books"]:
price = Decimal(entry['price'])
quantity = Decimal(entry['quantity'])
side = BID if entry['side'] == "buy" else ASK
if quantity == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
delta[side].append((price, 0))
else:
self._l2_book[pair].book[side][price] = quantity
delta[side].append((price, quantity))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
delta=delta)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Probit can send multiple type updates in one message so we avoid the use of elif
if 'recent_trades' in msg:
await self._trades(msg, timestamp)
if 'order_books' in msg:
await self._l2_update(msg, timestamp)
# Probit has a 'ticker' channel, but it provide OHLC-last data, not BBO px.
async def subscribe(self, conn: AsyncConnection):
self.__reset()
if self.subscription:
for chan in self.subscription:
for pair in self.subscription[chan]:
await conn.write(
json.dumps({
"type": "subscribe",
"channel": "marketdata",
"filter": [chan],
"interval": 100,
"market_id": pair,
}))
class BinanceFutures(Binance, BinanceFuturesRestMixin):
id = BINANCE_FUTURES
websocket_endpoints = [
WebsocketEndpoint('wss://fstream.binance.com',
sandbox='wss://stream.binancefuture.com',
options={'compression': None})
]
rest_endpoints = [
RestEndpoint('https://fapi.binance.com',
sandbox='https://testnet.binancefuture.com',
routes=Routes(
'/fapi/v1/exchangeInfo',
l2book='/fapi/v1/depth?symbol={}&limit={}',
authentication='/fapi/v1/listenKey',
open_interest='/fapi/v1//openInterest?symbol={}'))
]
valid_depths = [5, 10, 20, 50, 100, 500, 1000]
valid_depth_intervals = {'100ms', '250ms', '500ms'}
websocket_channels = {
**Binance.websocket_channels, FUNDING: 'markPrice',
OPEN_INTEREST: 'open_interest',
LIQUIDATIONS: 'forceOrder',
POSITIONS: POSITIONS
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
base, info = super()._parse_symbol_data(data)
add = {}
for symbol, orig in base.items():
if "_" in orig:
continue
add[f"{symbol.replace('PERP', 'PINDEX')}"] = f"p{orig}"
base.update(add)
return base, info
def __init__(self, open_interest_interval=1.0, **kwargs):
"""
open_interest_interval: float
time in seconds between open_interest polls
"""
super().__init__(**kwargs)
self.open_interest_interval = open_interest_interval
def _connect_rest(self):
ret = []
for chan in set(self.subscription):
if chan == 'open_interest':
addrs = [
self.rest_endpoints[0].route(
'open_interest', sandbox=self.sandbox).format(pair)
for pair in self.subscription[chan]
]
ret.append((HTTPPoll(addrs,
self.id,
delay=60.0,
sleep=self.open_interest_interval,
proxy=self.http_proxy), self.subscribe,
self.message_handler, self.authenticate))
return ret
def _check_update_id(self, pair: str, msg: dict) -> bool:
if self._l2_book[pair].delta is None and msg[
'u'] < self.last_update_id[pair]:
return True
elif msg['U'] <= self.last_update_id[pair] <= msg['u']:
self.last_update_id[pair] = msg['u']
return False
elif self.last_update_id[pair] == msg['pu']:
self.last_update_id[pair] = msg['u']
return False
else:
self._reset()
LOG.warning("%s: Missing book update detected, resetting book",
self.id)
return True
async def _open_interest(self, msg: dict, timestamp: float):
"""
{
"openInterest": "10659.509",
"symbol": "BTCUSDT",
"time": 1589437530011 // Transaction time
}
"""
pair = msg['symbol']
oi = msg['openInterest']
if oi != self._open_interest_cache.get(pair, None):
o = OpenInterest(self.id,
self.exchange_symbol_to_std_symbol(pair),
Decimal(oi),
self.timestamp_normalize(msg['time']),
raw=msg)
await self.callback(OPEN_INTEREST, o, timestamp)
self._open_interest_cache[pair] = oi
async def _account_update(self, msg: dict, timestamp: float):
"""
{
"e": "ACCOUNT_UPDATE", // Event Type
"E": 1564745798939, // Event Time
"T": 1564745798938 , // Transaction
"a": // Update Data
{
"m":"ORDER", // Event reason type
"B":[ // Balances
{
"a":"USDT", // Asset
"wb":"122624.12345678", // Wallet Balance
"cw":"100.12345678", // Cross Wallet Balance
"bc":"50.12345678" // Balance Change except PnL and Commission
},
{
"a":"BUSD",
"wb":"1.00000000",
"cw":"0.00000000",
"bc":"-49.12345678"
}
],
"P":[
{
"s":"BTCUSDT", // Symbol
"pa":"0", // Position Amount
"ep":"0.00000", // Entry Price
"cr":"200", // (Pre-fee) Accumulated Realized
"up":"0", // Unrealized PnL
"mt":"isolated", // Margin Type
"iw":"0.00000000", // Isolated Wallet (if isolated position)
"ps":"BOTH" // Position Side
}，
{
"s":"BTCUSDT",
"pa":"20",
"ep":"6563.66500",
"cr":"0",
"up":"2850.21200",
"mt":"isolated",
"iw":"13200.70726908",
"ps":"LONG"
},
{
"s":"BTCUSDT",
"pa":"-10",
"ep":"6563.86000",
"cr":"-45.04000000",
"up":"-1423.15600",
"mt":"isolated",
"iw":"6570.42511771",
"ps":"SHORT"
}
]
}
}
"""
for balance in msg['a']['B']:
b = Balance(self.id,
balance['a'],
Decimal(balance['wb']),
None,
raw=msg)
await self.callback(BALANCES, b, timestamp)
for position in msg['a']['P']:
p = Position(self.id,
self.exchange_symbol_to_std_symbol(position['s']),
Decimal(position['pa']),
Decimal(position['ep']),
position['ps'].lower(),
Decimal(position['up']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(POSITIONS, p, timestamp)
async def _order_update(self, msg: dict, timestamp: float):
"""
{
"e":"ORDER_TRADE_UPDATE", // Event Type
"E":1568879465651, // Event Time
"T":1568879465650, // Transaction Time
"o":
{
"s":"BTCUSDT", // Symbol
"c":"TEST", // Client Order Id
// special client order id:
// starts with "autoclose-": liquidation order
// "adl_autoclose": ADL auto close order
"S":"SELL", // Side
"o":"TRAILING_STOP_MARKET", // Order Type
"f":"GTC", // Time in Force
"q":"0.001", // Original Quantity
"p":"0", // Original Price
"ap":"0", // Average Price
"sp":"7103.04", // Stop Price. Please ignore with TRAILING_STOP_MARKET order
"x":"NEW", // Execution Type
"X":"NEW", // Order Status
"i":8886774, // Order Id
"l":"0", // Order Last Filled Quantity
"z":"0", // Order Filled Accumulated Quantity
"L":"0", // Last Filled Price
"N":"USDT", // Commission Asset, will not push if no commission
"n":"0", // Commission, will not push if no commission
"T":1568879465651, // Order Trade Time
"t":0, // Trade Id
"b":"0", // Bids Notional
"a":"9.91", // Ask Notional
"m":false, // Is this trade the maker side?
"R":false, // Is this reduce only
"wt":"CONTRACT_PRICE", // Stop Price Working Type
"ot":"TRAILING_STOP_MARKET", // Original Order Type
"ps":"LONG", // Position Side
"cp":false, // If Close-All, pushed with conditional order
"AP":"7476.89", // Activation Price, only puhed with TRAILING_STOP_MARKET order
"cr":"5.0", // Callback Rate, only puhed with TRAILING_STOP_MARKET order
"rp":"0" // Realized Profit of the trade
}
}
"""
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(msg['o']['s']),
str(msg['o']['i']),
BUY if msg['o']['S'].lower() == 'buy' else SELL,
msg['o']['x'],
LIMIT if msg['o']['o'].lower() == 'limit' else
MARKET if msg['o']['o'].lower() == 'market' else None,
Decimal(msg['o']['ap']) if
not Decimal.is_zero(Decimal(msg['o']['ap'])) else None,
Decimal(msg['o']['q']),
Decimal(msg['o']['q']) - Decimal(msg['o']['z']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Handle REST endpoint messages first
if 'openInterest' in msg:
return await self._open_interest(msg, timestamp)
# Handle account updates from User Data Stream
if self.requires_authentication:
msg_type = msg.get('e')
if msg_type == 'ACCOUNT_UPDATE':
await self._account_update(msg, timestamp)
elif msg_type == 'ORDER_TRADE_UPDATE':
await self._order_update(msg, timestamp)
return
# Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>}
# streamName is of format <symbol>@<channel>
pair, _ = msg['stream'].split('@', 1)
msg = msg['data']
pair = pair.upper()
msg_type = msg.get('e')
if msg_type == 'bookTicker':
await self._ticker(msg, timestamp)
elif msg_type == 'depthUpdate':
await self._book(msg, pair, timestamp)
elif msg_type == 'aggTrade':
await self._trade(msg, timestamp)
elif msg_type == 'forceOrder':
await self._liquidations(msg, timestamp)
elif msg_type == 'markPriceUpdate':
await self._funding(msg, timestamp)
elif msg['e'] == 'kline':
await self._candle(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class Bitflyer(Feed):
id = BITFLYER
websocket_endpoints = [
WebsocketEndpoint('wss://ws.lightstream.bitflyer.com/json-rpc')
]
rest_endpoints = [
RestEndpoint('https://api.bitflyer.com',
routes=Routes([
'/v1/getmarkets/eu', '/v1/getmarkets/usa',
'/v1/getmarkets', '/v1/markets', '/v1/markets/usa',
'/v1/markets/eu'
]))
]
websocket_channels = {
L2_BOOK: 'lightning_board_{}',
TRADES: 'lightning_executions_{}',
TICKER: 'lightning_ticker_{}'
}
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
for datum in entry:
stype = datum['market_type'].lower()
expiration = None
if stype == FUTURES:
base, quote = datum['product_code'][:3], datum[
'product_code'][3:6]
expiration = datum['product_code'][6:]
elif stype == FX:
_, base, quote = datum['product_code'].split("_")
else:
base, quote = datum['product_code'].split("_")
s = Symbol(base, quote, type=stype, expiry_date=expiration)
ret[s.normalized] = datum['product_code']
info['instrument_type'][s.normalized] = stype
return ret, info
def __reset(self):
self._l2_book = {}
async def _ticker(self, msg: dict, timestamp: float):
"""
{
"jsonrpc": "2.0",
"method": "channelMessage",
"params": {
"channel": "lightning_ticker_BTC_USD",
"message": {
"product_code": "BTC_USD",
"state": "RUNNING",
"timestamp":"2020-12-25T21:16:19.3661298Z",
"tick_id": 703768,
"best_bid": 24228.97,
"best_ask": 24252.89,
"best_bid_size": 0.4006,
"best_ask_size": 0.4006,
"total_bid_depth": 64.73938803,
"total_ask_depth": 51.99613815,
"market_bid_size": 0.0,
"market_ask_size": 0.0,
"ltp": 24382.25,
"volume": 241.953371650000,
"volume_by_product": 241.953371650000
}
}
}
"""
pair = self.exchange_symbol_to_std_symbol(
msg['params']['message']['product_code'])
bid = msg['params']['message']['best_bid']
ask = msg['params']['message']['best_ask']
t = Ticker(self.id,
pair,
bid,
ask,
self.timestamp_normalize(
msg['params']['message']['timestamp']),
raw=msg)
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{
"jsonrpc":"2.0",
"method":"channelMessage",
"params":{
"channel":"lightning_executions_BTC_JPY",
"message":[
{
"id":2084881071,
"side":"BUY",
"price":2509125.0,
"size":0.005,
"exec_date":"2020-12-25T21:36:22.8840579Z",
"buy_child_order_acceptance_id":"JRF20201225-213620-004123",
"sell_child_order_acceptance_id":"JRF20201225-213620-133314"
}
]
}
}
"""
pair = self.exchange_symbol_to_std_symbol(
msg['params']['channel'][21:])
for update in msg['params']['message']:
t = Trade(self.id,
pair,
BUY if update['side'] == 'BUY' else SELL,
update['size'],
update['price'],
self.timestamp_normalize(update['exec_date']),
raw=update)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
"""
{
"jsonrpc":"2.0",
"method":"channelMessage",
"params":{
"channel":"lightning_board_BTC_JPY",
"message":{
"mid_price":2534243.0,
"bids":[
],
"asks":[
{
"price":2534500.0,
"size":0.0
},
{
"price":2536101.0,
"size":0.0
}
]
}
}
}
"""
snapshot = msg['params']['channel'].startswith(
'lightning_board_snapshot')
if snapshot:
pair = msg['params']['channel'].split(
"lightning_board_snapshot")[1][1:]
else:
pair = msg['params']['channel'].split("lightning_board")[1][1:]
pair = self.exchange_symbol_to_std_symbol(pair)
# Ignore deltas until a snapshot is received
if pair not in self._l2_book and not snapshot:
return
delta = None
if snapshot:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
else:
delta = {BID: [], ASK: []}
data = msg['params']['message']
for side in ('bids', 'asks'):
s = BID if side == 'bids' else ASK
if snapshot:
self._l2_book[pair].book[side] = {
d['price']: d['size']
for d in data[side]
}
else:
for entry in data[side]:
if entry['size'] == 0:
if entry['price'] in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][entry['price']]
delta[s].append((entry['price'], Decimal(0.0)))
else:
self._l2_book[pair].book[side][
entry['price']] = entry['size']
delta[s].append((entry['price'], entry['size']))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
delta=delta)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['params']['channel'].startswith("lightning_ticker_"):
await self._ticker(msg, timestamp)
elif msg['params']['channel'].startswith('lightning_executions_'):
await self._trade(msg, timestamp)
elif msg['params']['channel'].startswith('lightning_board_'):
await self._book(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
for pair in self.subscription[chan]:
if chan.startswith('lightning_board'):
# need to subscribe to snapshots too if subscribed to L2_BOOKS
await conn.write(
json.dumps({
"method": "subscribe",
"params": {
"channel": f'lightning_board_snapshot_{pair}'
}
}))
await conn.write(
json.dumps({
"method": "subscribe",
"params": {
"channel": chan.format(pair)
}
}))
class Deribit(Feed, DeribitRestMixin):
id = DERIBIT
websocket_endpoints = [
WebsocketEndpoint('wss://www.deribit.com/ws/api/v2',
sandbox='wss://test.deribit.com/ws/api/v2')
]
rest_endpoints = [
RestEndpoint('https://www.deribit.com',
sandbox='https://test.deribit.com',
routes=Routes([
'/api/v2/public/get_instruments?currency=BTC',
'/api/v2/public/get_instruments?currency=ETH',
'/api/v2/public/get_instruments?currency=USDC',
'/api/v2/public/get_instruments?currency=SOL'
]))
]
websocket_channels = {
L1_BOOK: 'quote',
L2_BOOK: 'book',
TRADES: 'trades',
TICKER: 'ticker',
FUNDING: 'ticker',
OPEN_INTEREST: 'ticker',
LIQUIDATIONS: 'trades',
ORDER_INFO: 'user.orders',
FILLS: 'user.trades',
BALANCES: 'user.portfolio',
}
request_limit = 20
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def is_authenticated_channel(cls, channel: str) -> bool:
return channel in (ORDER_INFO, FILLS, BALANCES, L2_BOOK, TRADES,
TICKER)
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
currencies = []
for entry in data:
for e in entry['result']:
base = e['base_currency']
if base not in currencies:
currencies.append(base)
quote = e['quote_currency']
stype = e['kind'] if e[
'settlement_period'] != 'perpetual' else PERPETUAL
otype = e.get('option_type')
strike_price = e.get('strike')
strike_price = int(strike_price) if strike_price else None
expiry = e['expiration_timestamp'] / 1000
s = Symbol(base,
quote,
type=FUTURES if stype == 'future' else stype,
option_type=otype,
strike_price=strike_price,
expiry_date=expiry)
ret[s.normalized] = e['instrument_name']
info['tick_size'][s.normalized] = e['tick_size']
info['instrument_type'][s.normalized] = stype
for currency in currencies:
s = Symbol(currency, currency, type=CURRENCY)
ret[s.normalized] = currency
info['instrument_type'][s.normalized] = CURRENCY
return ret, info
def __reset(self):
self._open_interest_cache = {}
self._l2_book = {}
self.seq_no = {}
async def _trade(self, msg: dict, timestamp: float):
"""
{
"params":
{
"data":
[
{
"trade_seq": 933,
"trade_id": "9178",
"timestamp": 1550736299753,
"tick_direction": 3,
"price": 3948.69,
"instrument_name": "BTC-PERPETUAL",
"index_price": 3930.73,
"direction": "sell",
"amount": 10
}
],
"channel": "trades.BTC-PERPETUAL.raw"
},
"method": "subscription",
"jsonrpc": "2.0"
}
"""
for trade in msg["params"]["data"]:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(
trade["instrument_name"]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
self.timestamp_normalize(trade['timestamp']),
id=trade['trade_id'],
raw=trade)
await self.callback(TRADES, t, timestamp)
if 'liquidation' in trade:
liq = Liquidation(self.id,
self.exchange_symbol_to_std_symbol(
trade["instrument_name"]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
trade['trade_id'],
FILLED,
self.timestamp_normalize(trade['timestamp']),
raw=trade)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def _ticker(self, msg: dict, timestamp: float):
'''
{
"params" : {
"data" : {
"timestamp" : 1550652954406,
"stats" : {
"volume" : null,
"low" : null,
"high" : null
},
"state" : "open",
"settlement_price" : 3960.14,
"open_interest" : 0.12759952124659626,
"min_price" : 3943.21,
"max_price" : 3982.84,
"mark_price" : 3940.06,
"last_price" : 3906,
"instrument_name" : "BTC-PERPETUAL",
"index_price" : 3918.51,
"funding_8h" : 0.01520525,
"current_funding" : 0.00499954,
"best_bid_price" : 3914.97,
"best_bid_amount" : 40,
"best_ask_price" : 3996.61,
"best_ask_amount" : 50
},
"channel" : "ticker.BTC-PERPETUAL.raw"
},
"method" : "subscription",
"jsonrpc" : "2.0"}
'''
pair = self.exchange_symbol_to_std_symbol(
msg['params']['data']['instrument_name'])
ts = self.timestamp_normalize(msg['params']['data']['timestamp'])
t = Ticker(self.id,
pair,
Decimal(msg["params"]["data"]['best_bid_price']),
Decimal(msg["params"]["data"]['best_ask_price']),
ts,
raw=msg)
await self.callback(TICKER, t, timestamp)
if "current_funding" in msg["params"]["data"] and "funding_8h" in msg[
"params"]["data"]:
f = Funding(self.id,
pair,
Decimal(msg['params']['data']['mark_price']),
Decimal(msg["params"]["data"]["current_funding"]),
None,
ts,
raw=msg)
await self.callback(FUNDING, f, timestamp)
oi = msg['params']['data']['open_interest']
if pair in self._open_interest_cache and oi == self._open_interest_cache[
pair]:
return
self._open_interest_cache[pair] = oi
o = OpenInterest(self.id, pair, Decimal(oi), ts, raw=msg)
await self.callback(OPEN_INTEREST, o, timestamp)
async def _quote(self, quote: dict, timestamp: float):
book = L1Book(self.id,
self.exchange_symbol_to_std_symbol(
quote['instrument_name']),
Decimal(quote['best_bid_price']),
Decimal(quote['best_bid_amount']),
Decimal(quote['best_ask_price']),
Decimal(quote['best_ask_amount']),
self.timestamp_normalize(quote['timestamp']),
raw=quote)
await self.callback(L1_BOOK, book, timestamp)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
pub_channels = []
pri_channels = []
for chan in self.subscription:
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
for pair in self.subscription[chan]:
if self.exchange_channel_to_std(chan) == BALANCES:
pri_channels.append(f"{chan}.{pair}")
else:
pri_channels.append(f"{chan}.{pair}.raw")
else:
for pair in self.subscription[chan]:
if self.exchange_channel_to_std(chan) == L1_BOOK:
pub_channels.append(f"{chan}.{pair}")
else:
pub_channels.append(f"{chan}.{pair}.raw")
if pub_channels:
msg = {
"jsonrpc": "2.0",
"id": "101",
"method": "public/subscribe",
"params": {
"channels": pub_channels
}
}
LOG.debug(
f'{conn.uuid}: Subscribing to public channels with message {msg}'
)
await conn.write(json.dumps(msg))
if pri_channels:
msg = {
"jsonrpc": "2.0",
"id": "102",
"method": "private/subscribe",
"params": {
"scope": f"session:{conn.uuid}",
"channels": pri_channels
}
}
LOG.debug(
f'{conn.uuid}: Subscribing to private channels with message {msg}'
)
await conn.write(json.dumps(msg))
async def _book_snapshot(self, msg: dict, timestamp: float):
"""
{
'jsonrpc': '2.0',
'method': 'subscription',
'params': {
'channel': 'book.BTC-PERPETUAL.raw',
'data': {
'timestamp': 1598232105378,
'instrument_name': 'BTC-PERPETUAL',
'change_id': 21486665526, '
'bids': [['new', Decimal('11618.5'), Decimal('279310.0')], ..... ]
'asks': [[ ....... ]]
}
}
}
"""
ts = msg["params"]["data"]["timestamp"]
pair = self.exchange_symbol_to_std_symbol(
msg["params"]["data"]["instrument_name"])
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for _, price, amount in msg["params"]["data"]["bids"]
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for _, price, amount in msg["params"]["data"]["asks"]
}
self.seq_no[pair] = msg["params"]["data"]["change_id"]
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(ts),
sequence_number=msg["params"]["data"]["change_id"],
raw=msg)
async def _book_update(self, msg: dict, timestamp: float):
ts = msg["params"]["data"]["timestamp"]
pair = self.exchange_symbol_to_std_symbol(
msg["params"]["data"]["instrument_name"])
if msg['params']['data']['prev_change_id'] != self.seq_no[pair]:
LOG.warning("%s: Missing sequence number detected for %s", self.id,
pair)
LOG.warning("%s: Requesting book snapshot", self.id)
raise MissingSequenceNumber
self.seq_no[pair] = msg['params']['data']['change_id']
delta = {BID: [], ASK: []}
for action, price, amount in msg["params"]["data"]["bids"]:
if action != "delete":
self._l2_book[pair].book.bids[price] = Decimal(amount)
delta[BID].append((Decimal(price), Decimal(amount)))
else:
del self._l2_book[pair].book.bids[price]
delta[BID].append((Decimal(price), Decimal(amount)))
for action, price, amount in msg["params"]["data"]["asks"]:
if action != "delete":
self._l2_book[pair].book.asks[price] = amount
delta[ASK].append((Decimal(price), Decimal(amount)))
else:
del self._l2_book[pair].book.asks[price]
delta[ASK].append((Decimal(price), Decimal(amount)))
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(ts),
raw=msg,
delta=delta,
sequence_number=msg['params']['data']['change_id'])
async def message_handler(self, msg: str, conn, timestamp: float):
msg_dict = json.loads(msg, parse_float=Decimal)
if 'error' in msg_dict.keys():
LOG.error("%s: Received Error message: %s, Error code: %s",
conn.uuid, msg_dict['error']['message'],
msg_dict['error']['code'])
elif "result" in msg_dict:
result = msg_dict["result"]
if 'id' in msg_dict:
id = str(msg_dict["id"])
if id == "0":
LOG.debug("%s: Connected", conn.uuid)
elif id == '101':
LOG.info("%s: Subscribed to public channels", conn.uuid)
LOG.debug("%s: %s", conn.uuid, result)
elif id == '102':
LOG.info("%s: Subscribed to authenticated channels",
conn.uuid)
LOG.debug("%s: %s", conn.uuid, result)
elif id.startswith('auth') and "access_token" in result:
'''
Access token is another way to be authenticated while sending messages to Deribit.
In this implementation 'scope session' method is used instead of 'acces token' method.
'''
LOG.debug(f"{conn.uuid}: Access token received")
else:
LOG.warning("%s: Unknown id in message %s", conn.uuid,
msg_dict)
else:
LOG.warning("%s: Unknown 'result' message received: %s",
conn.uuid, msg_dict)
elif 'params' in msg_dict:
params = msg_dict['params']
if 'channel' in params:
channel = params['channel']
if "ticker" == channel.split(".")[0]:
await self._ticker(msg_dict, timestamp)
elif "trades" == channel.split(".")[0]:
await self._trade(msg_dict, timestamp)
elif "book" == channel.split(".")[0]:
# checking if we got full book or its update
# if it's update there is 'prev_change_id' field
if "prev_change_id" not in msg_dict["params"]["data"].keys(
):
await self._book_snapshot(msg_dict, timestamp)
elif "prev_change_id" in msg_dict["params"]["data"].keys():
await self._book_update(msg_dict, timestamp)
elif "quote" == channel.split(".")[0]:
await self._quote(params['data'], timestamp)
elif channel.startswith("user"):
await self._user_channels(conn, msg_dict, timestamp,
channel.split(".")[1])
else:
LOG.warning("%s: Unknown channel %s", conn.uuid, msg_dict)
else:
LOG.warning("%s: Unknown 'params' message received: %s",
conn.uuid, msg_dict)
else:
LOG.warning("%s: Unknown message in msg_dict: %s", conn.uuid,
msg_dict)
async def authenticate(self, conn: AsyncConnection):
if self.requires_authentication:
auth = self._auth(self.key_id, self.key_secret, conn.uuid)
LOG.debug(f"{conn.uuid}: Authenticating with message: {auth}")
await conn.write(json.dumps(auth))
def _auth(self, key_id, key_secret, session_id: str) -> str:
# https://docs.deribit.com/?python#authentication
timestamp = round(datetime.now().timestamp() * 1000)
nonce = f'xyz{str(timestamp)[-5:]}'
signature = hmac.new(bytes(key_secret, "latin-1"),
bytes('{}\n{}\n{}'.format(timestamp, nonce, ""),
"latin-1"),
digestmod=hashlib.sha256).hexdigest().lower()
auth = {
"jsonrpc": "2.0",
"id": f"auth_{session_id}",
"method": "public/auth",
"params": {
"grant_type": "client_signature",
"client_id": key_id,
"timestamp": timestamp,
"signature": signature,
"nonce": nonce,
"data": "",
"scope": f"session:{session_id}"
}
}
return auth
async def _user_channels(self, conn: AsyncConnection, msg: dict,
timestamp: float, subchan: str):
order_status = {
"open": OPEN,
"filled": FILLED,
"rejected": FAILED,
"cancelled": CANCELLED,
"untriggered": OPEN
}
order_types = {
"limit": LIMIT,
"market": MARKET,
"stop_limit": STOP_LIMIT,
"stop_market": STOP_MARKET
}
if 'data' in msg['params']:
data = msg['params']['data']
if subchan == 'portfolio':
'''
{
"params" : {
"data" : {
"total_pl" : 0.00000425,
"session_upl" : 0.00000425,
"session_rpl" : -2e-8,
"projected_maintenance_margin" : 0.00009141,
"projected_initial_margin" : 0.00012542,
"projected_delta_total" : 0.0043,
"portfolio_margining_enabled" : false,
"options_vega" : 0,
"options_value" : 0,
"options_theta" : 0,
"options_session_upl" : 0,
"options_session_rpl" : 0,
"options_pl" : 0,
"options_gamma" : 0,
"options_delta" : 0,
"margin_balance" : 0.2340038,
"maintenance_margin" : 0.00009141,
"initial_margin" : 0.00012542,
"futures_session_upl" : 0.00000425,
"futures_session_rpl" : -2e-8,
"futures_pl" : 0.00000425,
"estimated_liquidation_ratio" : 0.01822795,
"equity" : 0.2340038,
"delta_total" : 0.0043,
"currency" : "BTC",
"balance" : 0.23399957,
"available_withdrawal_funds" : 0.23387415,
"available_funds" : 0.23387838
},
"channel" : "user.portfolio.btc"
},
"method" : "subscription",
"jsonrpc" : "2.0"
}
'''
b = Balance(self.id,
data['currency'],
Decimal(data['balance']),
Decimal(data['balance']) -
Decimal(data['available_withdrawal_funds']),
raw=data)
await self.callback(BALANCES, b, timestamp)
elif subchan == 'orders':
'''
{
"params" : {
"data" : {
"time_in_force" : "good_til_cancelled",
"replaced" : false,
"reduce_only" : false,
"profit_loss" : 0,
"price" : 10502.52,
"post_only" : false,
"original_order_type" : "market",
"order_type" : "limit",
"order_state" : "open",
"order_id" : "5",
"max_show" : 200,
"last_update_timestamp" : 1581507423789,
"label" : "",
"is_liquidation" : false,
"instrument_name" : "BTC-PERPETUAL",
"filled_amount" : 0,
"direction" : "buy",
"creation_timestamp" : 1581507423789,
"commission" : 0,
"average_price" : 0,
"api" : false,
"amount" : 200
},
"channel" : "user.orders.BTC-PERPETUAL.raw"
},
"method" : "subscription",
"jsonrpc" : "2.0"
}
'''
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(
data['instrument_name']),
data["order_id"],
BUY if msg["side"] == 'Buy' else SELL,
order_status[data["order_state"]],
order_types[data['order_type']],
Decimal(data['price']),
Decimal(data['filled_amount']),
Decimal(data['amount']) - Decimal(data['cumQuantity']),
self.timestamp_normalize(data["last_update_timestamp"]),
raw=data)
await self.callback(ORDER_INFO, oi, timestamp)
elif subchan == 'trades':
'''
{
"params" : {
"data" : [
{
"trade_seq" : 30289432,
"trade_id" : "48079254",
"timestamp" : 1590484156350,
"tick_direction" : 0,
"state" : "filled",
"self_trade" : false,
"reduce_only" : false,
"price" : 8954,
"post_only" : false,
"order_type" : "market",
"order_id" : "4008965646",
"matching_id" : null,
"mark_price" : 8952.86,
"liquidity" : "T",
"instrument_name" : "BTC-PERPETUAL",
"index_price" : 8956.73,
"fee_currency" : "BTC",
"fee" : 0.00000168,
"direction" : "sell",
"amount" : 20
}]
}
}
'''
for entry in data:
symbol = self.exchange_symbol_to_std_symbol(
entry['instrument_name'])
f = Fill(self.id,
symbol,
SELL if entry['direction'] == 'sell' else BUY,
Decimal(entry['amount']),
Decimal(entry['price']),
Decimal(entry['fee']),
entry['trade_id'],
entry['order_id'],
entry['order_type'],
TAKER if entry['liquidity'] == 'T' else MAKER,
self.timestamp_normalize(entry['timestamp']),
raw=entry)
await self.callback(FILLS, f, timestamp)
else:
LOG.warning("%s: Unknown channel 'user.%s'", conn.uuid,
subchan)
else:
LOG.warning("%s: Unknown message %s'", conn.uuid, msg)
class Bithumb(Feed):
'''
Before you use this bithumb implementation, you should know that this is exchange's API is pretty terrible.
For some unknown reason, bithumb's api_info page lists all their KRW symbols as USDT. Probably because they bought
the exchange and copied everything but didn't bother to update the reference data.
We'll just assume that anything USDT is actually KRW. A search on their exchange page
shows that there is no USDT symbols available. Please be careful when referencing their api_info page
'''
id = BITHUMB
websocket_endpoints = [
WebsocketEndpoint('wss://pubwss.bithumb.com/pub/ws')
]
rest_endpoints = [
RestEndpoint('https://api.bithumb.com',
routes=Routes(
['/public/ticker/ALL_BTC', '/public/ticker/ALL_KRW']))
]
websocket_channels = {
# L2_BOOK: 'orderbookdepth', <-- technically the exchange supports orderbooks but it only provides orderbook deltas, there is
# no way to synchronize against a rest snapshot, nor request/obtain an orderbook via the websocket, so this isn't really useful
TRADES: 'transaction',
}
@classmethod
def timestamp_normalize(cls, ts: dt) -> float:
return (ts - timedelta(hours=9)).timestamp()
# Override symbol_mapping class method, because this bithumb is a very special case.
# There is no actual page in the API for reference info.
# Need to query the ticker endpoint by quote currency for that info
# To qeury the ticker endpoint, you need to know which quote currency you want. So far, seems like the exhcnage
# only offers KRW and BTC as quote currencies.
@classmethod
def symbol_mapping(cls, refresh=False) -> Dict:
if Symbols.populated(cls.id) and not refresh:
return Symbols.get(cls.id)[0]
try:
data = {}
for ep in cls.rest_endpoints[0].route('instruments'):
ret = cls.http_sync.read(ep, json=True, uuid=cls.id)
if 'BTC' in ep:
data['BTC'] = ret
else:
data['KRW'] = ret
syms, info = cls._parse_symbol_data(data)
Symbols.set(cls.id, syms, info)
return syms
except Exception as e:
LOG.error("%s: Failed to parse symbol information: %s",
cls.id,
str(e),
exc_info=True)
raise
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
for quote_curr, response in data.items():
bases = response['data']
for base_curr in bases.keys():
if base_curr == 'date':
continue
s = Symbol(base_curr, quote_curr)
ret[s.normalized] = f"{base_curr}_{quote_curr}"
info['instrument_type'][s.normalized] = s.type
return ret, info
def __init__(self, max_depth=30, **kwargs):
super().__init__(max_depth=max_depth, **kwargs)
async def _trades(self, msg: dict, rtimestamp: float):
'''
{
"type": "transaction",
"content": {
"list": [
{
"symbol": "BTC_KRW", // currency code
"buySellGb": "1", // type of contract (1: sale contract, 2: buy contract)
"contPrice": "10579000", // execution price
"contQty": "0.01", // number of contracts
"contAmt": "105790.00", // execution amount
"contDtm": "2020-01-29 12:24:18.830039", // Signing time
"updn": "dn" // comparison with the previous price: up-up, dn-down
}
]
}
}
'''
trades = msg.get('content', {}).get('list', [])
for trade in trades:
# API ref list uses '-', but market data returns '_'
symbol = self.exchange_symbol_to_std_symbol(trade['symbol'])
timestamp = self.timestamp_normalize(trade['contDtm'])
price = Decimal(trade['contPrice'])
quantity = Decimal(trade['contQty'])
side = BUY if trade['buySellGb'] == '2' else SELL
t = Trade(self.id,
symbol,
side,
quantity,
price,
timestamp,
raw=trade)
await self.callback(TRADES, t, rtimestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
msg_type = msg.get('type', None)
if msg_type == 'transaction':
await self._trades(msg, timestamp)
elif msg_type is None and msg.get('status', None) == '0000':
return
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
if self.subscription:
for chan in self.subscription:
await conn.write(
json.dumps({
"type":
chan,
"symbols":
[symbol for symbol in self.subscription[chan]]
# API ref list uses '-', but subscription requires '_'
}))
class Bitget(Feed):
id = BITGET
websocket_endpoints = [
WebsocketEndpoint('wss://ws.bitget.com/spot/v1/stream')
]
rest_endpoints = [
RestEndpoint('https://api.bitget.com',
routes=Routes('/api/spot/v1/public/products'))
]
valid_candle_intervals = {
'1m', '5m', '15m', '30m', '1h', '4h', '12h', '1d', '1w'
}
websocket_channels = {
L2_BOOK: 'books',
TRADES: 'trade',
TICKER: 'ticker',
CANDLES: 'candle'
}
request_limit = 20
@classmethod
def timestamp_normalize(cls, ts: int) -> float:
return ts / 1000
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data['data']:
"""
{
"baseCoin":"ALPHA",
"makerFeeRate":"0.001",
"maxTradeAmount":"0",
"minTradeAmount":"2",
"priceScale":"4",
"quantityScale":"4",
"quoteCoin":"USDT",
"status":"online",
"symbol":"ALPHAUSDT_SPBL",
"symbolName":"ALPHAUSDT",
"takerFeeRate":"0.001"
}
"""
sym = Symbol(entry['baseCoin'], entry['quoteCoin'])
info['instrument_type'][sym.normalized] = sym.type
ret[sym.normalized] = entry['symbolName']
return ret, info
def __reset(self):
self._l2_book = {}
async def _ticker(self, msg: dict, timestamp: float):
"""
{
'action': 'snapshot',
'arg': {
'instType': 'sp',
'channel': 'ticker',
'instId': 'BTCUSDT'
},
'data': [
{
'instId': 'BTCUSDT',
'last': '46572.07',
'open24h': '46414.54',
'high24h': '46767.30',
'low24h': '46221.11',
'bestBid': '46556.590000',
'bestAsk': '46565.670000',
'baseVolume': '1927.0855',
'quoteVolume': '89120317.8812',
'ts': 1649013100029,
'labeId': 0
}
]
}
"""
for entry in msg['data']:
# sometimes snapshots do not have bids/asks in them
if 'bestBid' not in entry or 'bestAsk' not in entry:
continue
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(entry['instId']),
Decimal(entry['bestBid']),
Decimal(entry['bestAsk']),
self.timestamp_normalize(entry['ts']),
raw=entry)
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'action': 'update',
'arg': {
'instType': 'sp',
'channel': 'trade',
'instId': 'BTCUSDT'
},
'data': [
['1649014224602', '46464.51', '0.0023', 'sell']
]
}
"""
for entry in msg['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['arg']['instId']),
SELL if entry[3] == 'sell' else BUY,
Decimal(entry[2]),
Decimal(entry[1]),
self.timestamp_normalize(int(entry[0])),
raw=entry)
await self.callback(TRADES, t, timestamp)
async def _candle(self, msg: dict, timestamp: float):
'''
{
'action': 'update',
'arg': {
'instType': 'sp',
'channel': 'candle1m',
'instId': 'BTCUSDT'
},
'data': [['1649014920000', '46434.2', '46437.98', '46434.2', '46437.98', '0.9469']]
}
'''
for entry in msg['data']:
t = Candle(self.id,
self.exchange_symbol_to_std_symbol(
msg['arg']['instId']),
self.timestamp_normalize(int(entry[0])),
self.timestamp_normalize(int(entry[0])) +
timedelta_str_to_sec(self.candle_interval),
self.candle_interval,
None,
Decimal(entry[1]),
Decimal(entry[4]),
Decimal(entry[2]),
Decimal(entry[3]),
Decimal(entry[5]),
None,
self.timestamp_normalize(int(entry[0])),
raw=entry)
await self.callback(CANDLES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
sym = self.exchange_symbol_to_std_symbol(msg['arg']['instId'])
data = msg['data'][0]
if msg['action'] == 'snapshot':
'''
{
'action': 'snapshot',
'arg': {
'instType': 'sp',
'channel': 'books',
'instId': 'BTCUSDT'
},
'data': [
{
'asks': [['46700.38', '0.0554'], ['46701.25', '0.0147'], ...
'bids': [['46686.68', '0.0032'], ['46684.75', '0.0161'], ...
'checksum': -393656186,
'ts': '1649021358917'
}
]
}
'''
bids = {
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}
asks = {
Decimal(price): Decimal(amount)
for price, amount in data['asks']
}
self._l2_book[sym] = OrderBook(self.id,
sym,
max_depth=self.max_depth,
bids=bids,
asks=asks)
await self.book_callback(L2_BOOK,
self._l2_book[sym],
timestamp,
checksum=data['checksum'],
timestamp=self.timestamp_normalize(
int(data['ts'])),
raw=msg)
else:
'''
{
'action': 'update',
'arg': {
'instType': 'sp',
'channel': 'books',
'instId': 'BTCUSDT'
},
'data': [
{
'asks': [['46701.25', '0'], ['46701.46', '0.0054'], ...
'bids': [['46687.67', '0.0531'], ['46686.22', '0'], ...
'checksum': -750266015,
'ts': '1649021359467'
}
]
}
'''
delta = {BID: [], ASK: []}
for side, key in ((BID, 'bids'), (ASK, 'asks')):
for price, size in data[key]:
price = Decimal(price)
size = Decimal(size)
delta[side].append((price, size))
if size == 0:
del self._l2_book[sym].book[side][price]
else:
self._l2_book[sym].book[side][price] = size
await self.book_callback(L2_BOOK,
self._l2_book[sym],
timestamp,
delta=delta,
checksum=data['checksum'],
timestamp=self.timestamp_normalize(
int(data['ts'])),
raw=msg)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'event' in msg:
# {'event': 'subscribe', 'arg': {'instType': 'sp', 'channel': 'ticker', 'instId': 'BTCUSDT'}}
if msg['event'] == 'subscribe':
return
if msg['event'] == 'error':
LOG.error('%s: Error from exchange: %s', conn.uuid, msg)
return
if msg['arg']['channel'] == 'books':
await self._book(msg, timestamp)
elif msg['arg']['channel'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['arg']['channel'] == 'trade':
await self._trade(msg, timestamp)
elif msg['arg']['channel'].startswith('candle'):
await self._candle(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
args = []
interval = self.candle_interval
if interval[-1] != 'm':
interval[-1] = interval[-1].upper()
for chan, symbols in conn.subscription.items():
for s in symbols:
d = {
'instType':
'SPBL' if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)) else 'SP',
'channel':
chan if chan != 'candle' else 'candle' + interval,
'instId':
s
}
args.append(d)
await conn.write(json.dumps({"op": "subscribe", "args": args}))
class Poloniex(Feed, PoloniexRestMixin):
id = POLONIEX
websocket_endpoints = [WebsocketEndpoint('wss://api2.poloniex.com')]
rest_endpoints = [
RestEndpoint('https://poloniex.com',
routes=Routes('/public?command=returnTicker'))
]
_channel_map = {}
websocket_channels = {
L2_BOOK: L2_BOOK,
TRADES: TRADES,
TICKER: 1002,
}
request_limit = 6
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
for symbol in data:
cls._channel_map[data[symbol]['id']] = symbol
std = symbol.replace("STR", "XLM")
quote, base = std.split("_")
s = Symbol(base, quote)
ret[s.normalized] = symbol
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
self.seq_no = {}
async def _ticker(self, msg: dict, timestamp: float):
"""
Format:
currencyPair, last, lowestAsk, highestBid, percentChange, baseVolume,
quoteVolume, isFrozen, 24hrHigh, 24hrLow, postOnly, maintenance mode
The postOnly field indicates that new orders posted to the market must be non-matching orders (no taker orders).
Any orders that would match will be rejected. Maintenance mode indicates that maintenace is being performed
and orders will be rejected
"""
pair_id, _, ask, bid, _, _, _, _, _, _, _, _ = msg
if pair_id not in self._channel_map:
# Ignore new trading pairs that are added during long running sessions
return
pair = self.exchange_symbol_to_std_symbol(self._channel_map[pair_id])
t = Ticker(self.id, pair, Decimal(bid), Decimal(ask), None, raw=msg)
await self.callback(TICKER, t, timestamp)
async def _book(self, msg: dict, chan_id: int, timestamp: float):
delta = {BID: [], ASK: []}
msg_type = msg[0][0]
pair = None
# initial update (i.e. snapshot)
if msg_type == 'i':
delta = None
pair = msg[0][1]['currencyPair']
pair = self.exchange_symbol_to_std_symbol(pair)
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
# 0 is asks, 1 is bids
order_book = msg[0][1]['orderBook']
for index, side in enumerate([ASK, BID]):
for key in order_book[index]:
amount = Decimal(order_book[index][key])
price = Decimal(key)
self._l2_book[pair].book[side][price] = amount
else:
pair = self._channel_map[chan_id]
pair = self.exchange_symbol_to_std_symbol(pair)
for update in msg:
msg_type = update[0]
# order book update
if msg_type == 'o':
side = ASK if update[1] == 0 else BID
price = Decimal(update[2])
amount = Decimal(update[3])
if amount == 0:
delta[side].append((price, 0))
del self._l2_book[pair].book[side][price]
else:
delta[side].append((price, amount))
self._l2_book[pair].book[side][price] = amount
elif msg_type == 't':
# index 1 is trade id, 2 is side, 3 is price, 4 is amount, 5 is timestamp, 6 is timestamp ms
_, order_id, _, price, amount, server_ts, _ = update
price = Decimal(price)
amount = Decimal(amount)
t = Trade(self.id,
pair,
BUY if update[2] == 1 else SELL,
amount,
price,
float(server_ts),
id=order_id,
raw=msg)
await self.callback(TRADES, t, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id,
msg)
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
delta=delta,
raw=msg)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'error' in msg:
LOG.error("%s: Error from exchange: %s", self.id, msg)
return
chan_id = msg[0]
if chan_id == 1002:
# the ticker channel doesn't have sequence ids
# so it should be None, except for the subscription
# ack, in which case its 1
seq_id = msg[1]
if seq_id is None and self._channel_map[
msg[2][0]] in self.subscription[1002]:
await self._ticker(msg[2], timestamp)
elif chan_id < 1000:
# order book updates - the channel id refers to
# the trading pair being updated
seq_no = msg[1]
if chan_id not in self.seq_no:
self.seq_no[chan_id] = seq_no
elif self.seq_no[chan_id] + 1 != seq_no and msg[2][0][0] != 'i':
LOG.warning(
"%s: missing sequence number. Received %d, expected %d",
self.id, seq_no, self.seq_no[chan_id] + 1)
raise MissingSequenceNumber
self.seq_no[chan_id] = seq_no
if msg[2][0][0] == 'i':
del self.seq_no[chan_id]
symbol = self._channel_map[chan_id]
if symbol in self._trade_book_symbols:
await self._book(msg[2], chan_id, timestamp)
elif chan_id == 1010:
# heartbeat - ignore
pass
else:
LOG.warning('%s: Invalid message type %s', self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
self._trade_book_symbols = []
for chan in self.subscription:
if chan == L2_BOOK or chan == TRADES:
for symbol in self.subscription[chan]:
if symbol in self._trade_book_symbols:
continue
self._trade_book_symbols.append(symbol)
await conn.write(
json.dumps({
"command": "subscribe",
"channel": symbol
}))
else:
await conn.write(
json.dumps({
"command": "subscribe",
"channel": chan
}))
self._trade_book_symbols = set(self._trade_book_symbols)
class Bittrex(Feed):
id = BITTREX
websocket_endpoints = [
WebsocketEndpoint('wss://www.bitmex.com/realtime', authentication=True)
]
rest_endpoints = [
RestEndpoint('https://api.bittrex.com',
routes=Routes('/v3/markets',
l2book='/v3/markets/{}/orderbook?depth={}'))
]
valid_candle_intervals = {'1m', '5m', '1h', '1d'}
valid_depths = [1, 25, 500]
websocket_channels = {
L2_BOOK: 'orderbook_{}_{}',
TRADES: 'trade_{}',
TICKER: 'ticker_{}',
CANDLES: 'candle_{}_{}'
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
info = {'instrument_type': {}}
ret = {}
for e in data:
if e['status'] != 'ONLINE':
continue
s = Symbol(e['baseCurrencySymbol'], e['quoteCurrencySymbol'])
ret[s.normalized] = e['symbol']
info['instrument_type'][s.normalized] = s.type
return ret, info
async def _ws_authentication(self, address: str,
options: dict) -> Tuple[str, dict]:
# Technically this isnt authentication, its the negotiation step for SignalR that
# we are performing here since this method is called right before connecting
r = self.http_sync.read(
'https://socket-v3.bittrex.com/signalr/negotiate',
params={
'connectionData': json.dumps([{
'name': 'c3'
}]),
'clientProtocol': 1.5
},
json=True)
token = r['ConnectionToken']
url = requests.Request('GET',
'https://socket-v3.bittrex.com/signalr/connect',
params={
'transport': 'webSockets',
'connectionToken': token,
'connectionData': json.dumps([{
"name": "c3"
}]),
'clientProtocol': 1.5
}).prepare().url
return url.replace('https://', 'wss://'), options
def __reset(self):
self._l2_book = {}
self.seq_no = {}
def __depth(self):
depth = self.valid_depths[-1]
if self.max_depth:
if 25 <= self.max_depth >= 500:
depth = 500
else:
depth = 25
return depth
async def ticker(self, msg: dict, timestamp: float):
"""
{
'symbol': 'BTC-USDT',
'lastTradeRate': '38904.35254113',
'bidRate': '38868.52330647',
'askRate': '38886.38815323'
}
"""
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
Decimal(msg['bidRate']),
Decimal(msg['askRate']),
None,
raw=msg)
await self.callback(TICKER, t, timestamp)
async def book(self, msg: dict, timestamp: float):
"""
{
'marketSymbol': 'BTC-USDT',
'depth': 500,
'sequence': 6032818,
'bidDeltas': [
{
'quantity': '0',
'rate': '38926.13088302'
},
{
'quantity': '0.00213516',
'rate': '31881.73000000'
}
],
'askDeltas': [
{
'quantity': '0.03106831',
'rate': '38989.50808432'
},
{
'quantity': '0.27954874',
'rate': '39013.57939993'
},
{
'quantity': '0',
'rate': '46667.67569819'
}
]
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['marketSymbol'])
seq_no = int(msg['sequence'])
delta = {BID: [], ASK: []}
if pair not in self._l2_book:
await self._snapshot(pair, seq_no)
else:
if seq_no <= self.seq_no[pair]:
return
if seq_no != self.seq_no[pair] + 1:
raise MissingSequenceNumber
self.seq_no[pair] = seq_no
for side, key in ((BID, 'bidDeltas'), (ASK, 'askDeltas')):
for update in msg[key]:
price = Decimal(update['rate'])
size = Decimal(update['quantity'])
if size == 0:
delta[side].append((price, 0))
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
else:
self._l2_book[pair].book[side][price] = size
delta[side].append((price, size))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
sequence_number=seq_no,
delta=delta)
async def _snapshot(self, symbol: str, sequence_number: int):
while True:
ret, headers = await self.http_conn.read(
self.rest_endpoints[0].route('l2book', self.sandbox).format(
symbol, self.__depth()),
return_headers=True)
seq = int(headers['Sequence'])
if seq >= sequence_number:
break
await asyncio.sleep(1.0)
self.seq_no[symbol] = seq
data = json.loads(ret, parse_float=Decimal)
self._l2_book[symbol] = OrderBook(self.id,
symbol,
max_depth=self.max_depth)
for side, entries in data.items():
self._l2_book[symbol].book[side] = {
Decimal(e['rate']): Decimal(e['quantity'])
for e in entries
}
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
time.time(),
raw=data,
sequence_number=seq)
async def trades(self, msg: dict, timestamp: float):
"""
{
'deltas': [
{
'id': '8e7f693b-6504-4cb7-9484-835435b147f9',
'executedAt': datetime.datetime(2021, 6, 13, 22, 38, 11, 80000, tzinfo=datetime.timezone.utc),
'quantity': '0.00693216',
'rate': '38808.83000000',
'takerSide': 'BUY'
}
],
'sequence': 204392,
'marketSymbol': 'BTC-USD'
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['marketSymbol'])
for trade in msg['deltas']:
t = Trade(self.id,
pair,
BUY if trade['takerSide'] == 'BUY' else SELL,
Decimal(trade['quantity']),
Decimal(trade['rate']),
self.timestamp_normalize(trade['executedAt']),
id=trade['id'],
raw=trade)
await self.callback(TRADES, t, timestamp)
async def candle(self, msg: dict, timestamp: float):
"""
{
'sequence': 134514,
'marketSymbol': 'BTC-USDT',
'interval': 'MINUTE_1',
'delta': {
'startsAt': datetime.datetime(2021, 6, 14, 1, 12, tzinfo=datetime.timezone.utc),
'open': '39023.31434847',
'high': '39023.31434847',
'low': '39023.31434847',
'close': '39023.31434847',
'volume': '0.05944473',
'quoteVolume': '2319.73038514'
},
'candleType': 'TRADE'
}
"""
start = self.timestamp_normalize(msg['delta']['startsAt'])
offset = 0
if self.candle_interval == '1m':
offset = 60
elif self.candle_interval == '5m':
offset = 300
elif self.candle_interval == '1h':
offset = 3600
elif self.candle_interval == '1d':
offset = 86400
end = start + offset
c = Candle(self.id,
self.exchange_symbol_to_std_symbol(msg['marketSymbol']),
start,
end,
self.candle_interval,
None,
Decimal(msg['delta']['open']),
Decimal(msg['delta']['close']),
Decimal(msg['delta']['high']),
Decimal(msg['delta']['low']),
Decimal(msg['delta']['volume']),
None,
None,
raw=msg)
await self.callback(CANDLES, c, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg)
if 'M' in msg and len(msg['M']) > 0:
for update in msg['M']:
if update['M'] == 'orderBook':
for message in update['A']:
data = json.loads(zlib.decompress(
base64.b64decode(message),
-zlib.MAX_WBITS).decode(),
parse_float=Decimal)
await self.book(data, timestamp)
elif update['M'] == 'trade':
for message in update['A']:
data = json.loads(zlib.decompress(
base64.b64decode(message),
-zlib.MAX_WBITS).decode(),
parse_float=Decimal)
await self.trades(data, timestamp)
elif update['M'] == 'ticker':
for message in update['A']:
data = json.loads(zlib.decompress(
base64.b64decode(message),
-zlib.MAX_WBITS).decode(),
parse_float=Decimal)
await self.ticker(data, timestamp)
elif update['M'] == 'candle':
for message in update['A']:
data = json.loads(zlib.decompress(
base64.b64decode(message),
-zlib.MAX_WBITS).decode(),
parse_float=Decimal)
await self.candle(data, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
elif 'E' in msg:
LOG.error("%s: Error from exchange %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
# H: Hub, M: Message, A: Args, I: Internal ID
# For more signalR info see:
# https://blog.3d-logic.com/2015/03/29/signalr-on-the-wire-an-informal-description-of-the-signalr-protocol/
# http://blogs.microsoft.co.il/applisec/2014/03/12/signalr-message-format/
for chan in self.subscription:
channel = self.exchange_channel_to_std(chan)
i = 1
for symbol in self.subscription[chan]:
if channel == L2_BOOK:
msg = {
'A': ([chan.format(symbol, self.__depth())], ),
'H': 'c3',
'I': i,
'M': 'Subscribe'
}
elif channel in (TRADES, TICKER):
msg = {
'A': ([chan.format(symbol)], ),
'H': 'c3',
'I': i,
'M': 'Subscribe'
}
elif channel == CANDLES:
interval = None
if self.candle_interval == '1m':
interval = 'MINUTE_1'
elif self.candle_interval == '5m':
interval = 'MINUTE_5'
elif self.candle_interval == '1h':
interval = 'HOUR_1'
elif self.candle_interval == '1d':
interval = 'DAY_1'
msg = {
'A': ([chan.format(symbol, interval)], ),
'H': 'c3',
'I': i,
'M': 'Subscribe'
}
else:
LOG.error("%s: invalid subscription for channel %s",
channel)
await conn.write(json.dumps(msg))
i += 1
class HuobiDM(Feed):
id = HUOBI_DM
websocket_endpoints = [WebsocketEndpoint('wss://www.hbdm.com/ws')]
rest_endpoints = [
RestEndpoint('https://www.hbdm.com',
routes=Routes('/api/v1/contract_contract_info'))
]
websocket_channels = {
L2_BOOK: 'depth.step0',
TRADES: 'trade.detail',
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for e in data['data']:
# Pricing is all in USD, see https://huobiglobal.zendesk.com/hc/en-us/articles/360000113102-Introduction-of-Huobi-Futures
s = Symbol(e['symbol'],
'USD',
type=FUTURES,
expiry_date=e['contract_code'].replace(e['symbol'], ''))
ret[s.normalized] = e['contract_code']
info['tick_size'][s.normalized] = e['price_tick']
info['instrument_type'][s.normalized] = FUTURES
return ret, info
def __reset(self):
self._l2_book = {}
async def _book(self, msg: dict, timestamp: float):
"""
{
'ch':'market.BTC_CW.depth.step0',
'ts':1565857755564,
'tick':{
'mrid':14848858327,
'id':1565857755,
'bids':[
[ Decimal('9829.99'), 1], ...
]
'asks':[
[ 9830, 625], ...
]
},
'ts':1565857755552,
'version':1565857755,
'ch':'market.BTC_CW.depth.step0'
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1])
data = msg['tick']
# When Huobi Delists pairs, empty updates still sent:
# {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951241196, 'tick': {'mrid': 50651100044, 'id': 1606951241, 'ts': 1606951241195, 'version': 1606951241, 'ch': 'market.AKRO-USD.depth.step0'}}
# {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951242297, 'tick': {'mrid': 50651100044, 'id': 1606951242, 'ts': 1606951242295, 'version': 1606951242, 'ch': 'market.AKRO-USD.depth.step0'}}
if 'bids' in data and 'asks' in data:
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for price, amount in data['asks']
}
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(
msg['ts']),
raw=msg)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'ch': 'market.btcusd.trade.detail',
'ts': 1549773923965,
'tick': {
'id': 100065340982,
'ts': 1549757127140,
'data': [{'id': '10006534098224147003732', 'amount': Decimal('0.0777'), 'price': Decimal('3669.69'), 'direction': 'buy', 'ts': 1549757127140}]}
}
"""
for trade in msg['tick']['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(
msg['ch'].split('.')[1]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
self.timestamp_normalize(trade['ts']),
id=str(trade['id']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
# unzip message
msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS)
msg = json.loads(msg, parse_float=Decimal)
# Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it
if 'ping' in msg:
await conn.write(json.dumps({'pong': msg['ping']}))
elif 'status' in msg and msg['status'] == 'ok':
return
elif 'ch' in msg:
if 'trade' in msg['ch']:
await self._trade(msg, timestamp)
elif 'depth' in msg['ch']:
await self._book(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
client_id = 0
for chan, symbols in conn.subscription.items():
for symbol in symbols:
client_id += 1
await conn.write(
json.dumps({
"sub": f"market.{symbol}.{chan}",
"id": str(client_id)
}))
class Huobi(Feed):
id = HUOBI
websocket_endpoints = [WebsocketEndpoint('wss://api.huobi.pro/ws')]
rest_endpoints = [
RestEndpoint('https://api.huobi.pro',
routes=Routes('/v1/common/symbols'))
]
valid_candle_intervals = {
'1m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '1M', '1Y'
}
candle_interval_map = {
'1m': '1min',
'5m': '5min',
'15m': '15min',
'30m': '30min',
'1h': '60min',
'4h': '4hour',
'1d': '1day',
'1M': '1mon',
'1w': '1week',
'1Y': '1year'
}
websocket_channels = {
L2_BOOK: 'depth.step0',
TRADES: 'trade.detail',
CANDLES: 'kline',
TICKER: 'ticker'
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
for e in data['data']:
if e['state'] == 'offline':
continue
base, quote = e['base-currency'].upper(
), e['quote-currency'].upper()
s = Symbol(base, quote)
ret[s.normalized] = e['symbol']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
async def _book(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1])
data = msg['tick']
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for price, amount in data['asks']
}
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(msg['ts']),
raw=msg)
async def _ticker(self, msg: dict, timestamp: float):
"""
{
"ch":"market.btcusdt.ticker",
"ts":1630982370526,
"tick":{
"open":51732,
"high":52785.64,
"low":51000,
"close":52735.63,
"amount":13259.24137056181,
"vol":687640987.4125315,
"count":448737,
"bid":52732.88,
"bidSize":0.036,
"ask":52732.89,
"askSize":0.583653,
"lastPrice":52735.63,
"lastSize":0.03
}
}
"""
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1]),
msg['tick']['bid'],
msg['tick']['ask'],
self.timestamp_normalize(msg['ts']),
raw=msg['tick'])
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'ch': 'market.adausdt.trade.detail',
'ts': 1597792835344,
'tick': {
'id': 101801945127,
'ts': 1597792835336,
'data': [
{
'id': Decimal('10180194512782291967181675'), <- per docs this is deprecated
'ts': 1597792835336,
'tradeId': 100341530602,
'amount': Decimal('0.1'),
'price': Decimal('0.137031'),
'direction': 'sell'
}
]
}
}
"""
for trade in msg['tick']['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(
msg['ch'].split('.')[1]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
self.timestamp_normalize(trade['ts']),
id=str(trade['tradeId']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _candles(self, msg: dict, symbol: str, interval: str,
timestamp: float):
"""
{
'ch': 'market.btcusdt.kline.1min',
'ts': 1618700872863,
'tick': {
'id': 1618700820,
'open': Decimal('60751.62'),
'close': Decimal('60724.73'),
'low': Decimal('60724.73'),
'high': Decimal('60751.62'),
'amount': Decimal('2.1990737759143966'),
'vol': Decimal('133570.944386'),
'count': 235}
}
}
"""
interval = self.normalize_candle_interval[interval]
start = int(msg['tick']['id'])
end = start + timedelta_str_to_sec(interval) - 1
c = Candle(self.id,
self.exchange_symbol_to_std_symbol(symbol),
start,
end,
interval,
msg['tick']['count'],
Decimal(msg['tick']['open']),
Decimal(msg['tick']['close']),
Decimal(msg['tick']['high']),
Decimal(msg['tick']['low']),
Decimal(msg['tick']['amount']),
None,
self.timestamp_normalize(msg['ts']),
raw=msg)
await self.callback(CANDLES, c, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
# unzip message
msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS)
msg = json.loads(msg, parse_float=Decimal)
# Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it
if 'ping' in msg:
await conn.write(json.dumps({'pong': msg['ping']}))
elif 'status' in msg and msg['status'] == 'ok':
return
elif 'ch' in msg:
if 'trade' in msg['ch']:
await self._trade(msg, timestamp)
elif 'tick' in msg['ch']:
await self._ticker(msg, timestamp)
elif 'depth' in msg['ch']:
await self._book(msg, timestamp)
elif 'kline' in msg['ch']:
_, symbol, _, interval = msg['ch'].split(".")
await self._candles(msg, symbol, interval, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
client_id = 0
for chan in self.subscription:
for pair in self.subscription[chan]:
client_id += 1
normalized_chan = self.exchange_channel_to_std(chan)
await conn.write(
json.dumps({
"sub":
f"market.{pair}.{chan}"
if normalized_chan != CANDLES else
f"market.{pair}.{chan}.{self.candle_interval_map[self.candle_interval]}",
"id":
client_id
}))
class OKX(Feed, OKXRestMixin):
id = OKX_str
websocket_channels = {
L2_BOOK: 'books',
TRADES: 'trades',
TICKER: 'tickers',
FUNDING: 'funding-rate',
OPEN_INTEREST: 'open-interest',
LIQUIDATIONS: LIQUIDATIONS,
ORDER_INFO: 'orders',
}
websocket_endpoints = [
WebsocketEndpoint('wss://ws.okx.com:8443/ws/v5/public',
channel_filter=(websocket_channels[L2_BOOK],
websocket_channels[TRADES],
websocket_channels[TICKER],
websocket_channels[FUNDING],
websocket_channels[OPEN_INTEREST],
websocket_channels[LIQUIDATIONS]),
options={'compression': None}),
WebsocketEndpoint('wss://ws.okx.com:8443/ws/v5/private',
channel_filter=(websocket_channels[ORDER_INFO], ),
options={'compression': None}),
]
rest_endpoints = [
RestEndpoint(
'https://www.okx.com',
routes=Routes(
[
'/api/v5/public/instruments?instType=SPOT',
'/api/v5/public/instruments?instType=SWAP',
'/api/v5/public/instruments?instType=FUTURES',
'/api/v5/public/instruments?instType=OPTION&uly=BTC-USD',
'/api/v5/public/instruments?instType=OPTION&uly=ETH-USD'
],
liquidations=
'/api/v5/public/liquidation-orders?instType={}&limit=100&state={}&uly={}'
))
]
request_limit = 20
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
for e in entry['data']:
expiry = None
otype = None
stype = e['instType'].lower()
strike = None
if stype == SPOT:
base = e['baseCcy']
quote = e['quoteCcy']
elif stype == FUTURES:
base, quote, expiry = e['instId'].split("-")
elif stype == OPTION:
base, quote, expiry, strike, otype = e['instId'].split("-")
otype = PUT if otype == 'P' else CALL
elif stype == 'swap':
# this is a perpetual swap (aka perpetual futures contract), not a real swap
stype = PERPETUAL
base, quote, _ = e['instId'].split("-")
s = Symbol(base,
quote,
expiry_date=expiry,
type=stype,
option_type=otype,
strike_price=strike)
ret[s.normalized] = e['instId']
info['tick_size'][s.normalized] = e['tickSz']
info['instrument_type'][s.normalized] = stype
return ret, info
async def _liquidations(self, pairs: list):
last_update = defaultdict(dict)
"""
for PERP liquidations, the following arguments are required: uly, state
for FUTURES liquidations, the following arguments are required: uly, state, alias
FUTURES, MARGIN and OPTION liquidation request not currently supported by the below
"""
while True:
for pair in pairs:
if 'SWAP' in pair:
instrument_type = 'SWAP'
uly = pair.split("-")[0] + "-" + pair.split("-")[1]
else:
continue
for status in (FILLED, UNFILLED):
data = await self.http_conn.read(
self.rest_endpoints[0].route(
'liquidations', sandbox=self.sandbox).format(
instrument_type, status, uly))
data = json.loads(data, parse_float=Decimal)
timestamp = time.time()
if len(data['data'][0]['details']) == 0 or (
len(data['data'][0]['details']) > 0
and last_update.get(pair)
== data['data'][0]['details'][0]):
continue
for entry in data['data'][0]['details']:
if pair in last_update:
if entry == last_update[pair].get(status):
break
liq = Liquidation(
self.id,
pair,
BUY if entry['side'] == 'buy' else SELL,
Decimal(entry['sz']),
Decimal(entry['bkPx']),
None,
status,
self.timestamp_normalize(int(entry['ts'])),
raw=data)
await self.callback(LIQUIDATIONS, liq, timestamp)
last_update[pair][status] = data['data'][0]['details'][0]
await asyncio.sleep(0.1)
await asyncio.sleep(60)
def __reset(self):
self._l2_book = {}
@classmethod
def instrument_type(cls, symbol: str):
return cls.info()['instrument_type'][symbol]
async def _ticker(self, msg: dict, timestamp: float):
"""
{"arg": {"channel": "tickers", "instId": "LTC-USD-200327"}, "data": [{"instType": "SWAP","instId": "LTC-USD-SWAP","last": "9999.99","lastSz": "0.1","askPx": "9999.99","askSz": "11","bidPx": "8888.88","bidSz": "5","open24h": "9000","high24h": "10000","low24h": "8888.88","volCcy24h": "2222","vol24h": "2222","sodUtc0": "2222","sodUtc8": "2222","ts": "1597026383085"}]}
"""
pair = self.exchange_symbol_to_std_symbol(msg['arg']['instId'])
for update in msg['data']:
update_timestamp = self.timestamp_normalize(int(update['ts']))
t = Ticker(
self.id,
pair,
Decimal(update['bidPx']) if update['bidPx'] else Decimal(0),
Decimal(update['askPx']) if update['askPx'] else Decimal(0),
update_timestamp,
raw=update)
await self.callback(TICKER, t, timestamp)
async def _open_interest(self, msg: dict, timestamp: float):
"""
{
'arg': {
'channel': 'open-interest',
'instId': 'BTC-USDT-SWAP
},
'data': [
{
'instId': 'BTC-USDT-SWAP',
'instType': 'SWAP',
'oi':'565474',
'oiCcy': '5654.74',
'ts': '1630338003010'
}
]
}
"""
symbol = self.exchange_symbol_to_std_symbol(msg['arg']['instId'])
for update in msg['data']:
oi = OpenInterest(self.id,
symbol,
Decimal(update['oi']),
self.timestamp_normalize(int(update['ts'])),
raw=update)
await self.callback(OPEN_INTEREST, oi, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{
"arg": {
"channel": "trades",
"instId": "BTC-USD-191227"
},
"data": [
{
"instId": "BTC-USD-191227",
"tradeId": "9",
"px": "0.016",
"sz": "50",
"side": "buy",
"ts": "1597026383085"
}
]
}
"""
for trade in msg['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(trade['instId']),
BUY if trade['side'] == 'buy' else SELL,
Decimal(trade['sz']),
Decimal(trade['px']),
self.timestamp_normalize(int(trade['ts'])),
id=trade['tradeId'],
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _funding(self, msg: dict, timestamp: float):
for update in msg['data']:
f = Funding(self.id,
self.exchange_symbol_to_std_symbol(update['instId']),
None,
Decimal(update['fundingRate']),
None,
self.timestamp_normalize(int(update['fundingTime'])),
predicted_rate=Decimal(update['nextFundingRate']),
raw=update)
await self.callback(FUNDING, f, timestamp)
async def _book(self, msg: dict, timestamp: float):
if msg['action'] == 'snapshot':
# snapshot
pair = self.exchange_symbol_to_std_symbol(msg['arg']['instId'])
for update in msg['data']:
bids = {
Decimal(price): Decimal(amount)
for price, amount, *_ in update['bids']
}
asks = {
Decimal(price): Decimal(amount)
for price, amount, *_ in update['asks']
}
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth,
checksum_format=self.id,
bids=bids,
asks=asks)
if self.checksum_validation and self._l2_book[
pair].book.checksum() != (update['checksum']
& 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(int(update['ts'])),
checksum=update['checksum'] & 0xFFFFFFFF,
raw=msg)
else:
# update
pair = self.exchange_symbol_to_std_symbol(msg['arg']['instId'])
for update in msg['data']:
delta = {BID: [], ASK: []}
for side in ('bids', 'asks'):
s = BID if side == 'bids' else ASK
for price, amount, *_ in update[side]:
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
if price in self._l2_book[pair].book[s]:
delta[s].append((price, 0))
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, amount))
self._l2_book[pair].book[s][price] = amount
if self.checksum_validation and self._l2_book[
pair].book.checksum() != (update['checksum']
& 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(int(update['ts'])),
raw=msg,
delta=delta,
checksum=update['checksum'] & 0xFFFFFFFF)
async def _order(self, msg: dict, timestamp: float):
'''
{
"arg": {
"channel": "orders",
"instType": "FUTURES",
"instId": "BTC-USD-200329"
},
"data": [
{
"instType": "FUTURES",
"instId": "BTC-USD-200329",
"ccy": "BTC",
"ordId": "312269865356374016",
"clOrdId": "b1",
"tag": "",
"px": "999",
"sz": "333",
"notionalUsd": "",
"ordType": "limit",
"side": "buy",
"posSide": "long",
"tdMode": "cross",
"tgtCcy": "",
"fillSz": "0",
"fillPx": "long",
"tradeId": "0",
"accFillSz": "323",
"fillNotionalUsd": "",
"fillTime": "0",
"fillFee": "0.0001",
"fillFeeCcy": "BTC",
"execType": "T",
"state": "canceled",
"avgPx": "0",
"lever": "20",
"tpTriggerPx": "0",
"tpOrdPx": "20",
"slTriggerPx": "0",
"slOrdPx": "20",
"feeCcy": "",
"fee": "",
"rebateCcy": "",
"rebate": "",
"tgtCcy":"",
"pnl": "",
"category": "",
"uTime": "1597026383085",
"cTime": "1597026383085",
"reqId": "",
"amendResult": "",
"code": "0",
"msg": ""
}
]
}
'''
status = msg['data'][0]['state']
if status == 'canceled':
status == CANCELLED
elif status == 'live':
status == OPEN
elif status == 'partially-filled':
status = PARTIAL
elif status == 'filled':
status = FILLED
o_type = msg['data'][0]['ordType']
if o_type == 'market':
o_type = MARKET
elif o_type == 'post_only':
o_type = MAKER_OR_CANCEL
elif o_type == 'fok':
o_type = FILL_OR_KILL
elif o_type == 'ioc':
o_type = IMMEDIATE_OR_CANCEL
elif o_type == 'limit':
o_type = LIMIT
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(
msg['data'][0]['instId'].upper()),
msg['data'][0]['ordId'],
BUY if msg['data'][0]['side'].lower() == 'buy' else SELL,
status,
o_type,
Decimal(msg['data'][0]['px'])
if msg['data'][0]['px'] else Decimal(msg['data'][0]['avgPx']),
Decimal(msg['data'][0]['sz']),
Decimal(msg['data'][0]['sz']) -
Decimal(msg['data'][0]['accFillSz'])
if msg['data'][0]['accFillSz'] else Decimal(msg['data'][0]['sz']),
self.timestamp_normalize(int(msg['data'][0]['uTime'])),
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def _login(self, msg: dict, timestamp: float):
LOG.debug('%s: Websocket logged in? %s', self.id, msg['code'])
async def message_handler(self, msg: str, conn, timestamp: float):
# DEFLATE compression, no header
# msg = zlib.decompress(msg, -15)
# not required, as websocket now set to "Per-Message Deflate"
msg = json.loads(msg, parse_float=Decimal)
if 'event' in msg:
if msg['event'] == 'error':
LOG.error("%s: Error: %s", self.id, msg)
elif msg['event'] == 'subscribe':
pass
elif msg['event'] == 'login':
await self._login(msg, timestamp)
else:
LOG.warning("%s: Unhandled event %s", self.id, msg)
elif 'arg' in msg:
if self.websocket_channels[L2_BOOK] in msg['arg']['channel']:
await self._book(msg, timestamp)
elif self.websocket_channels[TICKER] in msg['arg']['channel']:
await self._ticker(msg, timestamp)
elif self.websocket_channels[TRADES] in msg['arg']['channel']:
await self._trade(msg, timestamp)
elif self.websocket_channels[FUNDING] in msg['arg']['channel']:
await self._funding(msg, timestamp)
elif self.websocket_channels[ORDER_INFO] in msg['arg']['channel']:
await self._order(msg, timestamp)
elif self.websocket_channels[OPEN_INTEREST] in msg['arg'][
'channel']:
await self._open_interest(msg, timestamp)
else:
LOG.warning("%s: Unhandled message %s", self.id, msg)
async def subscribe(self, connection: AsyncConnection):
channels = []
for chan in self.subscription:
if chan == LIQUIDATIONS:
asyncio.create_task(self._liquidations(
self.subscription[chan]))
continue
for pair in self.subscription[chan]:
channels.append(self.build_subscription(chan, pair))
msg = {"op": "subscribe", "args": channels}
await connection.write(json.dumps(msg))
async def authenticate(self, conn: AsyncConnection):
if self.requires_authentication:
if any([
self.is_authenticated_channel(
self.exchange_channel_to_std(chan))
for chan in conn.subscription
]):
auth = self._auth(self.key_id, self.key_secret)
LOG.debug(f"{conn.uuid}: Authenticating with message: {auth}")
await conn.write(json.dumps(auth))
await asyncio.sleep(1)
def _auth(self, key_id, key_secret) -> str:
timestamp, sign = self._generate_token(key_id, key_secret)
login_param = {
"op":
"login",
"args": [{
"apiKey": self.key_id,
"passphrase": self.key_passphrase,
"timestamp": timestamp,
"sign": sign.decode("utf-8")
}]
}
return login_param
def build_subscription(self, channel: str, ticker: str) -> dict:
if channel in ['positions', 'orders']:
subscription_dict = {
"channel": channel,
"instType": self.inst_type_to_okx_type(ticker),
"instId": ticker
}
else:
subscription_dict = {"channel": channel, "instId": ticker}
return subscription_dict
def inst_type_to_okx_type(self, ticker):
sym = self.exchange_symbol_to_std_symbol(ticker)
instrument_type = self.instrument_type(sym)
instrument_type_map = {
'perpetual': 'SWAP',
'spot': 'MARGIN',
'futures': 'FUTURES',
'option': 'OPTION'
}
return instrument_type_map.get(instrument_type, 'MARGIN')
def _get_server_time(self):
endpoint = "public/time"
response = requests.get(self.api + endpoint)
if response.status_code == 200:
return response.json()['data'][0]['ts']
else:
return ""
def _server_timestamp(self):
server_time = self._get_server_time()
return int(server_time) / 1000
def _create_sign(self, timestamp: str, key_secret: str):
message = timestamp + 'GET' + '/users/self/verify'
mac = hmac.new(bytes(key_secret, encoding='utf8'),
bytes(message, encoding='utf-8'),
digestmod='sha256')
d = mac.digest()
sign = base64.b64encode(d)
return sign
def _generate_token(self, key_id: str, key_secret: str) -> dict:
timestamp = str(self._server_timestamp())
sign = self._create_sign(timestamp, key_secret)
return timestamp, sign
class Coinbase(Feed, CoinbaseRestMixin):
id = COINBASE
websocket_endpoints = [WebsocketEndpoint('wss://ws-feed.pro.coinbase.com', options={'compression': None})]
rest_endpoints = [RestEndpoint('https://api.pro.coinbase.com', routes=Routes('/products', l3book='/products/{}/book?level=3'))]
websocket_channels = {
L2_BOOK: 'level2',
L3_BOOK: 'full',
TRADES: 'matches',
TICKER: 'ticker',
}
request_limit = 10
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
sym = Symbol(entry['base_currency'], entry['quote_currency'])
info['tick_size'][sym.normalized] = entry['quote_increment']
info['instrument_type'][sym.normalized] = sym.type
ret[sym.normalized] = entry['id']
return ret, info
def __init__(self, callbacks=None, **kwargs):
super().__init__(callbacks=callbacks, **kwargs)
# we only keep track of the L3 order book if we have at least one subscribed order-book callback.
# use case: subscribing to the L3 book plus Trade type gives you order_type information (see _received below),
# and we don't need to do the rest of the book-keeping unless we have an active callback
self.keep_l3_book = False
if callbacks and L3_BOOK in callbacks:
self.keep_l3_book = True
self.__reset()
def __reset(self):
self.order_map = {}
self.order_type_map = {}
self.seq_no = None
# sequence number validation only works when the FULL data stream is enabled
chan = self.std_channel_to_exchange(L3_BOOK)
if chan in self.subscription:
pairs = self.subscription[chan]
self.seq_no = {pair: None for pair in pairs}
self._l3_book = {}
self._l2_book = {}
async def _ticker(self, msg: dict, timestamp: float):
'''
{
'type': 'ticker',
'sequence': 5928281084,
'product_id': 'BTC-USD',
'price': '8500.01000000',
'open_24h': '8217.24000000',
'volume_24h': '4529.1293778',
'low_24h': '8172.00000000',
'high_24h': '8600.00000000',
'volume_30d': '329178.93594133',
'best_bid': '8500',
'best_ask': '8500.01'
}
{
'type': 'ticker',
'sequence': 5928281348,
'product_id': 'BTC-USD',
'price': '8500.00000000',
'open_24h': '8217.24000000',
'volume_24h': '4529.13179472',
'low_24h': '8172.00000000',
'high_24h': '8600.00000000',
'volume_30d': '329178.93835825',
'best_bid': '8500',
'best_ask': '8500.01',
'side': 'sell',
'time': '2018-05-21T00:30:11.587000Z',
'trade_id': 43736677,
'last_size': '0.00241692'
}
'''
await self.callback(TICKER, Ticker(self.id, self.exchange_symbol_to_std_symbol(msg['product_id']), Decimal(msg['best_bid']), Decimal(msg['best_ask']), self.timestamp_normalize(msg['time']), raw=msg), timestamp)
async def _book_update(self, msg: dict, timestamp: float):
'''
{
'type': 'match', or last_match
'trade_id': 43736593
'maker_order_id': '2663b65f-b74e-4513-909d-975e3910cf22',
'taker_order_id': 'd058d737-87f1-4763-bbb4-c2ccf2a40bde',
'side': 'buy',
'size': '0.01235647',
'price': '8506.26000000',
'product_id': 'BTC-USD',
'sequence': 5928276661,
'time': '2018-05-21T00:26:05.585000Z'
}
'''
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
ts = self.timestamp_normalize(msg['time'])
if self.keep_l3_book and 'full' in self.subscription and pair in self.subscription['full']:
delta = {BID: [], ASK: []}
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
size = Decimal(msg['size'])
maker_order_id = msg['maker_order_id']
_, new_size = self.order_map[maker_order_id]
new_size -= size
if new_size <= 0:
del self.order_map[maker_order_id]
self.order_type_map.pop(maker_order_id, None)
delta[side].append((maker_order_id, price, 0))
del self._l3_book[pair].book[side][price][maker_order_id]
if len(self._l3_book[pair].book[side][price]) == 0:
del self._l3_book[pair].book[side][price]
else:
self.order_map[maker_order_id] = (price, new_size)
self._l3_book[pair].book[side][price][maker_order_id] = new_size
delta[side].append((maker_order_id, price, new_size))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=ts, delta=delta, raw=msg, sequence_number=self.seq_no[pair])
order_type = self.order_type_map.get(msg['taker_order_id'])
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(msg['product_id']),
SELL if msg['side'] == 'buy' else BUY,
Decimal(msg['size']),
Decimal(msg['price']),
ts,
id=str(msg['trade_id']),
type=order_type,
raw=msg
)
await self.callback(TRADES, t, timestamp)
async def _pair_level2_snapshot(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
bids = {Decimal(price): Decimal(amount) for price, amount in msg['bids']}
asks = {Decimal(price): Decimal(amount) for price, amount in msg['asks']}
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, bids=bids, asks=asks)
else:
self._l2_book[pair].book.bids = bids
self._l2_book[pair].book.asks = asks
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg)
async def _pair_level2_update(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
ts = self.timestamp_normalize(msg['time'])
delta = {BID: [], ASK: []}
for side, price, amount in msg['changes']:
side = BID if side == 'buy' else ASK
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
del self._l2_book[pair].book[side][price]
delta[side].append((price, 0))
else:
self._l2_book[pair].book[side][price] = amount
delta[side].append((price, amount))
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=ts, raw=msg, delta=delta)
async def _book_snapshot(self, pairs: list):
# Coinbase needs some time to send messages to us
# before we request the snapshot. If we don't sleep
# the snapshot seq no could be much earlier than
# the subsequent messages, causing a seq no mismatch.
await asyncio.sleep(2)
urls = [self.rest_endpoints[0].route('l3book', self.sandbox).format(pair) for pair in pairs]
results = []
for url in urls:
ret = await self.http_conn.read(url)
results.append(ret)
# rate limit - 3 per second
await asyncio.sleep(0.3)
timestamp = time.time()
for res, pair in zip(results, pairs):
orders = json.loads(res, parse_float=Decimal)
npair = self.exchange_symbol_to_std_symbol(pair)
self._l3_book[npair] = OrderBook(self.id, pair, max_depth=self.max_depth)
self.seq_no[npair] = orders['sequence']
for side in (BID, ASK):
for price, size, order_id in orders[side + 's']:
price = Decimal(price)
size = Decimal(size)
if price in self._l3_book[npair].book[side]:
self._l3_book[npair].book[side][price][order_id] = size
else:
self._l3_book[npair].book[side][price] = {order_id: size}
self.order_map[order_id] = (price, size)
await self.book_callback(L3_BOOK, self._l3_book[npair], timestamp, raw=orders)
async def _open(self, msg: dict, timestamp: float):
if not self.keep_l3_book:
return
delta = {BID: [], ASK: []}
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
size = Decimal(msg['remaining_size'])
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
order_id = msg['order_id']
ts = self.timestamp_normalize(msg['time'])
if price in self._l3_book[pair].book[side]:
self._l3_book[pair].book[side][price][order_id] = size
else:
self._l3_book[pair].book[side][price] = {order_id: size}
self.order_map[order_id] = (price, size)
delta[side].append((order_id, price, size))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=ts, delta=delta, raw=msg, sequence_number=msg['sequence'])
async def _done(self, msg: dict, timestamp: float):
"""
per Coinbase API Docs:
A done message will be sent for received orders which are fully filled or canceled due
to self-trade prevention. There will be no open message for such orders. Done messages
for orders which are not on the book should be ignored when maintaining a real-time order book.
"""
if 'price' not in msg:
# market order life cycle: received -> done
self.order_type_map.pop(msg['order_id'], None)
self.order_map.pop(msg['order_id'], None)
return
order_id = msg['order_id']
self.order_type_map.pop(order_id, None)
if order_id not in self.order_map:
return
del self.order_map[order_id]
if self.keep_l3_book:
delta = {BID: [], ASK: []}
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
ts = self.timestamp_normalize(msg['time'])
del self._l3_book[pair].book[side][price][order_id]
if len(self._l3_book[pair].book[side][price]) == 0:
del self._l3_book[pair].book[side][price]
delta[side].append((order_id, price, 0))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, delta=delta, timestamp=ts, raw=msg, sequence_number=msg['sequence'])
async def _received(self, msg: dict, timestamp: float):
"""
per Coinbase docs:
A valid order has been received and is now active. This message is emitted for every single
valid order as soon as the matching engine receives it whether it fills immediately or not.
This message is the only time we receive the order type (limit vs market) for a given order,
so we keep it in a map by order ID.
"""
order_id = msg["order_id"]
order_type = msg["order_type"]
self.order_type_map[order_id] = order_type
async def _change(self, msg: dict, timestamp: float):
"""
Like done, these updates can be sent for orders that are not in the book. Per the docs:
Not all done or change messages will result in changing the order book. These messages will
be sent for received orders which are not yet on the order book. Do not alter
the order book for such messages, otherwise your order book will be incorrect.
"""
if not self.keep_l3_book:
return
delta = {BID: [], ASK: []}
if 'price' not in msg or not msg['price']:
return
order_id = msg['order_id']
if order_id not in self.order_map:
return
ts = self.timestamp_normalize(msg['time'])
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
new_size = Decimal(msg['new_size'])
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
self._l3_book[pair].book[side][price][order_id] = new_size
self.order_map[order_id] = (price, new_size)
delta[side].append((order_id, price, new_size))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, delta=delta, timestamp=ts, raw=msg, sequence_number=msg['sequence'])
async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float):
# PERF perf_start(self.id, 'msg')
msg = json.loads(msg, parse_float=Decimal)
if self.seq_no:
if 'product_id' in msg and 'sequence' in msg:
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
if not self.seq_no.get(pair, None):
return
if msg['sequence'] <= self.seq_no[pair]:
return
if msg['sequence'] != self.seq_no[pair] + 1:
LOG.warning("%s: Missing sequence number detected for %s. Received %d, expected %d", self.id, pair, msg['sequence'], self.seq_no[pair] + 1)
LOG.warning("%s: Resetting data for %s", self.id, pair)
self.__reset(symbol=pair)
await self._book_snapshot([pair])
return
self.seq_no[pair] = msg['sequence']
if 'type' in msg:
if msg['type'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['type'] == 'match' or msg['type'] == 'last_match':
await self._book_update(msg, timestamp)
elif msg['type'] == 'snapshot':
await self._pair_level2_snapshot(msg, timestamp)
elif msg['type'] == 'l2update':
await self._pair_level2_update(msg, timestamp)
elif msg['type'] == 'open':
await self._open(msg, timestamp)
elif msg['type'] == 'done':
await self._done(msg, timestamp)
elif msg['type'] == 'change':
await self._change(msg, timestamp)
elif msg['type'] == 'received':
await self._received(msg, timestamp)
elif msg['type'] == 'activate':
pass
elif msg['type'] == 'subscriptions':
pass
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
# PERF perf_end(self.id, 'msg')
# PERF perf_log(self.id, 'msg')
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
await conn.write(json.dumps({"type": "subscribe",
"product_ids": self.subscription[chan],
"channels": [chan]
}))
chan = self.std_channel_to_exchange(L3_BOOK)
if chan in self.subscription:
await self._book_snapshot(self.subscription[chan])
class Binance(Feed, BinanceRestMixin):
id = BINANCE
websocket_endpoints = [
WebsocketEndpoint('wss://stream.binance.com:9443',
sandbox='wss://testnet.binance.vision')
]
rest_endpoints = [
RestEndpoint('https://api.binance.com',
routes=Routes('/api/v3/exchangeInfo',
l2book='/api/v3/depth?symbol={}&limit={}',
authentication='/api/v3/userDataStream'),
sandbox='https://testnet.binance.vision')
]
valid_depths = [5, 10, 20, 50, 100, 500, 1000, 5000]
# m -> minutes; h -> hours; d -> days; w -> weeks; M -> months
valid_candle_intervals = {
'1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h',
'1d', '3d', '1w', '1M'
}
valid_depth_intervals = {'100ms', '1000ms'}
websocket_channels = {
L2_BOOK: 'depth',
TRADES: 'aggTrade',
TICKER: 'bookTicker',
CANDLES: 'kline_',
BALANCES: BALANCES,
ORDER_INFO: ORDER_INFO
}
request_limit = 20
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for symbol in data['symbols']:
if symbol.get('status', 'TRADING') != "TRADING":
continue
if symbol.get('contractStatus', 'TRADING') != "TRADING":
continue
expiration = None
stype = SPOT
if symbol.get('contractType') == 'PERPETUAL':
stype = PERPETUAL
elif symbol.get('contractType') in ('CURRENT_QUARTER',
'NEXT_QUARTER'):
stype = FUTURES
expiration = symbol['symbol'].split("_")[1]
s = Symbol(symbol['baseAsset'],
symbol['quoteAsset'],
type=stype,
expiry_date=expiration)
ret[s.normalized] = symbol['symbol']
info['tick_size'][s.normalized] = symbol['filters'][0]['tickSize']
info['instrument_type'][s.normalized] = stype
return ret, info
def __init__(self, depth_interval='100ms', **kwargs):
"""
depth_interval: str
time between l2_book/delta updates {'100ms', '1000ms'} (different from BINANCE_FUTURES & BINANCE_DELIVERY)
"""
if depth_interval is not None and depth_interval not in self.valid_depth_intervals:
raise ValueError(
f"Depth interval must be one of {self.valid_depth_intervals}")
super().__init__(**kwargs)
self.depth_interval = depth_interval
self._open_interest_cache = {}
self._reset()
def _address(self) -> Union[str, Dict]:
"""
Binance has a 200 pair/stream limit per connection, so we need to break the address
down into multiple connections if necessary. Because the key is currently not used
for the address dict, we can just set it to the last used stream, since this will be
unique.
The generic connect method supplied by Feed will take care of creating the
correct connection objects from the addresses.
"""
if self.requires_authentication:
listen_key = self._generate_token()
address = self.address
address += '/ws/' + listen_key
else:
address = self.address
address += '/stream?streams='
subs = []
is_any_private = any(
self.is_authenticated_channel(chan) for chan in self.subscription)
is_any_public = any(not self.is_authenticated_channel(chan)
for chan in self.subscription)
if is_any_private and is_any_public:
raise ValueError(
"Private channels should be subscribed in separate feeds vs public channels"
)
if all(
self.is_authenticated_channel(chan)
for chan in self.subscription):
return address
for chan in self.subscription:
normalized_chan = self.exchange_channel_to_std(chan)
if normalized_chan == OPEN_INTEREST:
continue
if self.is_authenticated_channel(normalized_chan):
continue
stream = chan
if normalized_chan == CANDLES:
stream = f"{chan}{self.candle_interval}"
elif normalized_chan == L2_BOOK:
stream = f"{chan}@{self.depth_interval}"
for pair in self.subscription[chan]:
# for everything but premium index the symbols need to be lowercase.
if pair.startswith("p"):
if normalized_chan != CANDLES:
raise ValueError(
"Premium Index Symbols only allowed on Candle data feed"
)
else:
pair = pair.lower()
subs.append(f"{pair}@{stream}")
if 0 < len(subs) < 200:
return address + '/'.join(subs)
else:
def split_list(_list: list, n: int):
for i in range(0, len(_list), n):
yield _list[i:i + n]
return [
address + '/'.join(chunk) for chunk in split_list(subs, 200)
]
def _reset(self):
self._l2_book = {}
self.last_update_id = {}
async def _refresh_token(self):
while True:
await sleep(30 * 60)
if self._auth_token is None:
raise ValueError('There is no token to refresh')
payload = {'listenKey': self._auth_token}
r = requests.put(
f'{self.rest_endpoints[0].route("authentication", sandbox=self.sandbox)}?{urlencode(payload)}',
headers={'X-MBX-APIKEY': self.key_id})
r.raise_for_status()
def _generate_token(self) -> str:
url = self.rest_endpoints[0].route('authentication',
sandbox=self.sandbox)
r = requests.post(url, headers={'X-MBX-APIKEY': self.key_id})
r.raise_for_status()
response = r.json()
if 'listenKey' in response:
self._auth_token = response['listenKey']
return self._auth_token
else:
raise ValueError(f'Unable to retrieve listenKey token from {url}')
async def _trade(self, msg: dict, timestamp: float):
"""
{
"e": "aggTrade", // Event type
"E": 123456789, // Event time
"s": "BNBBTC", // Symbol
"a": 12345, // Aggregate trade ID
"p": "0.001", // Price
"q": "100", // Quantity
"f": 100, // First trade ID
"l": 105, // Last trade ID
"T": 123456785, // Trade time
"m": true, // Is the buyer the market maker?
"M": true // Ignore
}
"""
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['s']),
SELL if msg['m'] else BUY,
Decimal(msg['q']),
Decimal(msg['p']),
self.timestamp_normalize(msg['T']),
id=str(msg['a']),
raw=msg)
await self.callback(TRADES, t, timestamp)
async def _ticker(self, msg: dict, timestamp: float):
"""
{
'u': 382569232,
's': 'FETUSDT',
'b': '0.36031000',
'B': '1500.00000000',
'a': '0.36092000',
'A': '176.40000000'
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['s'])
bid = Decimal(msg['b'])
ask = Decimal(msg['a'])
# Binance does not have a timestamp in this update, but the two futures APIs do
if 'E' in msg:
ts = self.timestamp_normalize(msg['E'])
else:
ts = timestamp
t = Ticker(self.id, pair, bid, ask, ts, raw=msg)
await self.callback(TICKER, t, timestamp)
async def _liquidations(self, msg: dict, timestamp: float):
"""
{
"e":"forceOrder", // Event Type
"E":1568014460893, // Event Time
"o":{
"s":"BTCUSDT", // Symbol
"S":"SELL", // Side
"o":"LIMIT", // Order Type
"f":"IOC", // Time in Force
"q":"0.014", // Original Quantity
"p":"9910", // Price
"ap":"9910", // Average Price
"X":"FILLED", // Order Status
"l":"0.014", // Order Last Filled Quantity
"z":"0.014", // Order Filled Accumulated Quantity
"T":1568014460893, // Order Trade Time
}
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['o']['s'])
liq = Liquidation(self.id,
pair,
SELL if msg['o']['S'] == 'SELL' else BUY,
Decimal(msg['o']['q']),
Decimal(msg['o']['p']),
None,
FILLED if msg['o']['X'] == 'FILLED' else UNFILLED,
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(LIQUIDATIONS, liq, receipt_timestamp=timestamp)
def _check_update_id(self, std_pair: str, msg: dict) -> bool:
"""
Messages will be queued while fetching snapshot and we can return a book_callback
using this msg's data instead of waiting for the next update.
"""
if self._l2_book[std_pair].delta is None and msg[
'u'] <= self.last_update_id[std_pair]:
return True
elif msg['U'] <= self.last_update_id[std_pair] and msg[
'u'] <= self.last_update_id[std_pair]:
# Old message, can ignore it
return True
elif msg['U'] <= self.last_update_id[std_pair] + 1 <= msg['u']:
self.last_update_id[std_pair] = msg['u']
return False
elif self.last_update_id[std_pair] + 1 == msg['U']:
self.last_update_id[std_pair] = msg['u']
return False
else:
self._reset()
LOG.warning("%s: Missing book update detected, resetting book",
self.id)
return True
async def _snapshot(self, pair: str) -> None:
max_depth = self.max_depth if self.max_depth else 1000
if max_depth not in self.valid_depths:
for d in self.valid_depths:
if d > max_depth:
max_depth = d
break
resp = await self.http_conn.read(self.rest_endpoints[0].route(
'l2book', self.sandbox).format(pair, max_depth))
resp = json.loads(resp, parse_float=Decimal)
timestamp = self.timestamp_normalize(
resp['E']) if 'E' in resp else None
std_pair = self.exchange_symbol_to_std_symbol(pair)
self.last_update_id[std_pair] = resp['lastUpdateId']
self._l2_book[std_pair] = OrderBook(
self.id,
std_pair,
max_depth=self.max_depth,
bids={Decimal(u[0]): Decimal(u[1])
for u in resp['bids']},
asks={Decimal(u[0]): Decimal(u[1])
for u in resp['asks']})
await self.book_callback(L2_BOOK,
self._l2_book[std_pair],
time.time(),
timestamp=timestamp,
raw=resp,
sequence_number=self.last_update_id[std_pair])
async def _book(self, msg: dict, pair: str, timestamp: float):
"""
{
"e": "depthUpdate", // Event type
"E": 123456789, // Event time
"s": "BNBBTC", // Symbol
"U": 157, // First update ID in event
"u": 160, // Final update ID in event
"b": [ // Bids to be updated
[
"0.0024", // Price level to be updated
"10" // Quantity
]
],
"a": [ // Asks to be updated
[
"0.0026", // Price level to be updated
"100" // Quantity
]
]
}
"""
exchange_pair = pair
pair = self.exchange_symbol_to_std_symbol(pair)
if pair not in self._l2_book:
await self._snapshot(exchange_pair)
skip_update = self._check_update_id(pair, msg)
if skip_update:
return
delta = {BID: [], ASK: []}
for s, side in (('b', BID), ('a', ASK)):
for update in msg[s]:
price = Decimal(update[0])
amount = Decimal(update[1])
delta[side].append((price, amount))
if amount == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
else:
self._l2_book[pair].book[side][price] = amount
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(msg['E']),
raw=msg,
delta=delta,
sequence_number=self.last_update_id[pair])
async def _funding(self, msg: dict, timestamp: float):
"""
{
"e": "markPriceUpdate", // Event type
"E": 1562305380000, // Event time
"s": "BTCUSDT", // Symbol
"p": "11185.87786614", // Mark price
"r": "0.00030000", // Funding rate
"T": 1562306400000 // Next funding time
}
BinanceFutures
{
"e": "markPriceUpdate", // Event type
"E": 1562305380000, // Event time
"s": "BTCUSDT", // Symbol
"p": "11185.87786614", // Mark price
"i": "11784.62659091" // Index price
"P": "11784.25641265", // Estimated Settle Price, only useful in the last hour before the settlement starts
"r": "0.00030000", // Funding rate
"T": 1562306400000 // Next funding time
}
"""
next_time = self.timestamp_normalize(
msg['T']) if msg['T'] > 0 else None
rate = Decimal(msg['r']) if msg['r'] else None
if next_time is None:
rate = None
f = Funding(self.id,
self.exchange_symbol_to_std_symbol(msg['s']),
Decimal(msg['p']),
rate,
next_time,
self.timestamp_normalize(msg['E']),
predicted_rate=Decimal(msg['P'])
if 'P' in msg and msg['P'] is not None else None,
raw=msg)
await self.callback(FUNDING, f, timestamp)
async def _candle(self, msg: dict, timestamp: float):
"""
{
'e': 'kline',
'E': 1615927655524,
's': 'BTCUSDT',
'k': {
't': 1615927620000,
'T': 1615927679999,
's': 'BTCUSDT',
'i': '1m',
'f': 710917276,
'L': 710917780,
'o': '56215.99000000',
'c': '56232.07000000',
'h': '56238.59000000',
'l': '56181.99000000',
'v': '13.80522200',
'n': 505,
'x': False,
'q': '775978.37383076',
'V': '7.19660600',
'Q': '404521.60814919',
'B': '0'
}
}
"""
if self.candle_closed_only and not msg['k']['x']:
return
c = Candle(self.id,
self.exchange_symbol_to_std_symbol(msg['s']),
msg['k']['t'] / 1000,
msg['k']['T'] / 1000,
msg['k']['i'],
msg['k']['n'],
Decimal(msg['k']['o']),
Decimal(msg['k']['c']),
Decimal(msg['k']['h']),
Decimal(msg['k']['l']),
Decimal(msg['k']['v']),
msg['k']['x'],
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(CANDLES, c, timestamp)
async def _account_update(self, msg: dict, timestamp: float):
"""
{
"e": "outboundAccountPosition", //Event type
"E": 1564034571105, //Event Time
"u": 1564034571073, //Time of last account update
"B": [ //Balances Array
{
"a": "ETH", //Asset
"f": "10000.000000", //Free
"l": "0.000000" //Locked
}
]
}
"""
for balance in msg['B']:
b = Balance(self.id,
balance['a'],
Decimal(balance['f']),
Decimal(balance['l']),
raw=msg)
await self.callback(BALANCES, b, timestamp)
async def _order_update(self, msg: dict, timestamp: float):
"""
{
"e": "executionReport", // Event type
"E": 1499405658658, // Event time
"s": "ETHBTC", // Symbol
"c": "mUvoqJxFIILMdfAW5iGSOW", // Client order ID
"S": "BUY", // Side
"o": "LIMIT", // Order type
"f": "GTC", // Time in force
"q": "1.00000000", // Order quantity
"p": "0.10264410", // Order price
"P": "0.00000000", // Stop price
"F": "0.00000000", // Iceberg quantity
"g": -1, // OrderListId
"C": "", // Original client order ID; This is the ID of the order being canceled
"x": "NEW", // Current execution type
"X": "NEW", // Current order status
"r": "NONE", // Order reject reason; will be an error code.
"i": 4293153, // Order ID
"l": "0.00000000", // Last executed quantity
"z": "0.00000000", // Cumulative filled quantity
"L": "0.00000000", // Last executed price
"n": "0", // Commission amount
"N": null, // Commission asset
"T": 1499405658657, // Transaction time
"t": -1, // Trade ID
"I": 8641984, // Ignore
"w": true, // Is the order on the book?
"m": false, // Is this trade the maker side?
"M": false, // Ignore
"O": 1499405658657, // Order creation time
"Z": "0.00000000", // Cumulative quote asset transacted quantity
"Y": "0.00000000", // Last quote asset transacted quantity (i.e. lastPrice * lastQty)
"Q": "0.00000000" // Quote Order Qty
}
"""
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(msg['s']),
str(msg['i']),
BUY if msg['S'].lower() == 'buy' else SELL,
msg['x'],
LIMIT if msg['o'].lower() == 'limit' else
MARKET if msg['o'].lower() == 'market' else None,
Decimal(msg['Z']) / Decimal(msg['z'])
if not Decimal.is_zero(Decimal(msg['z'])) else None,
Decimal(msg['q']),
Decimal(msg['q']) - Decimal(msg['z']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Handle account updates from User Data Stream
if self.requires_authentication:
msg_type = msg['e']
if msg_type == 'outboundAccountPosition':
await self._account_update(msg, timestamp)
elif msg_type == 'executionReport':
await self._order_update(msg, timestamp)
return
# Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>}
# streamName is of format <symbol>@<channel>
pair, _ = msg['stream'].split('@', 1)
msg = msg['data']
pair = pair.upper()
if 'e' in msg:
if msg['e'] == 'depthUpdate':
await self._book(msg, pair, timestamp)
elif msg['e'] == 'aggTrade':
await self._trade(msg, timestamp)
elif msg['e'] == 'forceOrder':
await self._liquidations(msg, timestamp)
elif msg['e'] == 'markPriceUpdate':
await self._funding(msg, timestamp)
elif msg['e'] == 'kline':
await self._candle(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id,
msg)
elif 'A' in msg:
await self._ticker(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
# Binance does not have a separate subscribe message, the
# subscription information is included in the
# connection endpoint
if isinstance(conn, (HTTPPoll, HTTPConcurrentPoll)):
self._open_interest_cache = {}
else:
self._reset()
if self.requires_authentication:
create_task(self._refresh_token())
class BitDotCom(Feed):
id = BITDOTCOM
websocket_endpoints = [
WebsocketEndpoint('wss://spot-ws.bit.com',
instrument_filter=('TYPE', (SPOT, )),
sandbox='wss://betaspot-ws.bitexch.dev'),
WebsocketEndpoint('wss://ws.bit.com',
instrument_filter=('TYPE', (FUTURES, OPTION,
PERPETUAL)),
sandbox='wss://betaws.bitexch.dev'),
]
rest_endpoints = [
RestEndpoint('https://spot-api.bit.com',
instrument_filter=('TYPE', (SPOT, )),
sandbox='https://betaspot-api.bitexch.dev',
routes=Routes('/spot/v1/instruments',
authentication='/spot/v1/ws/auth')),
RestEndpoint('https://api.bit.com',
instrument_filter=('TYPE', (OPTION, FUTURES, PERPETUAL)),
sandbox='https://betaapi.bitexch.dev',
routes=Routes('/v1/instruments?currency={}&active=true',
currencies='/v1/currencies',
authentication='/v1/ws/auth'))
]
websocket_channels = {
L2_BOOK: 'depth',
TRADES: 'trade',
TICKER: 'ticker',
ORDER_INFO: 'order',
BALANCES: 'account',
FILLS: 'user_trade',
# funding rates paid and received
}
request_limit = 10
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._sequence_no = defaultdict(int)
@classmethod
def _symbol_endpoint_prepare(cls,
ep: RestEndpoint) -> Union[List[str], str]:
if ep.routes.currencies:
ret = cls.http_sync.read(ep.route('currencies'),
json=True,
uuid=cls.id)
return [
ep.route('instruments').format(currency)
for currency in ret['data']['currencies']
]
return ep.route('instruments')
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
if entry['code'] != 0:
raise ValueError('%s - Failed to collect instrument data - %s',
cls.id, entry['message'])
for mapping in entry['data']:
if 'category' in mapping:
expiry = None
strike = None
otype = None
if mapping['category'] == 'option':
stype = OPTION
strike = int(float(mapping['strike_price']))
expiry = cls.timestamp_normalize(
mapping['expiration_at'])
otype = mapping['option_type']
elif mapping['category'] == 'future':
if 'PERPETUAL' in mapping['instrument_id']:
stype = PERPETUAL
else:
stype = FUTURES
expiry = cls.timestamp_normalize(
mapping['expiration_at'])
s = Symbol(mapping['base_currency'],
mapping['quote_currency'],
type=stype,
option_type=otype,
expiry_date=expiry,
strike_price=strike)
ret[s.normalized] = mapping['instrument_id']
info['instrument_type'][s.normalized] = stype
else:
# Spot
s = Symbol(mapping['base_currency'],
mapping['quote_currency'],
type=SPOT)
ret[s.normalized] = mapping['pair']
info['instrument_type'][s.normalized] = SPOT
return ret, info
def __reset(self, conn: AsyncConnection):
if self.std_channel_to_exchange(L2_BOOK) in conn.subscription:
for pair in conn.subscription[self.std_channel_to_exchange(
L2_BOOK)]:
std_pair = self.exchange_symbol_to_std_symbol(pair)
if std_pair in self._l2_book:
del self._l2_book[std_pair]
if std_pair in self._sequence_no:
del self._sequence_no[std_pair]
def encode_list(self, item_list: list):
list_val = []
for item in item_list:
obj_val = self.encode_object(item)
list_val.append(obj_val)
output = '&'.join(list_val)
return '[' + output + ']'
def get_signature(self, api_path: str, param_map: dict):
str_to_sign = api_path + '&' + self.encode_object(param_map)
return hmac.new(self.key_secret.encode('utf-8'),
str_to_sign.encode('utf-8'),
digestmod=hashlib.sha256).hexdigest()
def encode_object(self, param_map: dict):
sorted_keys = sorted(param_map.keys())
ret_list = []
for key in sorted_keys:
val = param_map[key]
if isinstance(val, list):
list_val = self.encode_list(val)
ret_list.append(f'{key}={list_val}')
elif isinstance(val, dict):
dict_val = self.encode_object(val)
ret_list.append(f'{key}={dict_val}')
elif isinstance(val, bool):
bool_val = str(val).lower()
ret_list.append(f'{key}={bool_val}')
else:
general_val = str(val)
ret_list.append(f'{key}={general_val}')
sorted_list = sorted(ret_list)
return '&'.join(sorted_list)
async def authenticate(self, connection: AsyncConnection):
if not self.key_id or not self.key_secret:
return
if any([
self.is_authenticated_channel(self.exchange_channel_to_std(c))
for c in connection.subscription
]):
symbols = list(
set(itertools.chain(*connection.subscription.values())))
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(symbols[0]))
for ep in self.rest_endpoints:
if sym.type in ep.instrument_filter[1]:
ts = int(round(time.time() * 1000))
signature = self.get_signature(ep.routes.authentication,
{'timestamp': ts})
params = {'timestamp': ts, 'signature': signature}
ret = self.http_sync.read(
ep.route('authentication', sandbox=self.sandbox),
params=params,
headers={'X-Bit-Access-Key': self.key_id},
json=True)
if ret['code'] != 0 or 'token' not in ret['data']:
LOG.warning('%s: authentication failed: %s', ret)
token = ret['data']['token']
self._auth_token = token
return
async def subscribe(self, connection: AsyncConnection):
self.__reset(connection)
for chan, symbols in connection.subscription.items():
if len(symbols) == 0:
continue
stype = str_to_symbol(
self.exchange_symbol_to_std_symbol(symbols[0])).type
msg = {
'type': 'subscribe',
'channels': [chan],
'instruments' if stype in {PERPETUAL, FUTURES, OPTION} else 'pairs':
symbols,
}
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
msg['token'] = self._auth_token
await connection.write(json.dumps(msg))
async def _trade(self, data: dict, timestamp: float):
"""
{
'channel': 'trade',
'timestamp': 1639080717242,
'data': [{
'trade_id': '7016884324',
'instrument_id': 'BTC-PERPETUAL',
'price': '47482.50000000',
'qty': '6000.00000000',
'side': 'sell',
'sigma': '0.00000000',
'is_block_trade': False,
'created_at': 1639080717195
}]
}
"""
for t in data['data']:
trade = Trade(self.id,
self.exchange_symbol_to_std_symbol(
t.get('instrument_id') or t.get('pair')),
SELL if t['side'] == 'sell' else BUY,
Decimal(t['qty']),
Decimal(t['price']),
self.timestamp_normalize(t['created_at']),
id=t['trade_id'],
raw=t)
await self.callback(TRADES, trade, timestamp)
async def _book(self, data: dict, timestamp: float):
'''
Snapshot
{
'channel': 'depth',
'timestamp': 1639083660346,
'data': {
'type': 'snapshot',
'instrument_id': 'BTC-PERPETUAL',
'sequence': 1639042602148589825,
'bids': [
['47763.00000000', '20000.00000000'],
['47762.50000000', '6260.00000000'],
...
]
'asks': [
['47768.00000000', '10000.00000000'],
['47776.50000000', '20000.00000000'],
...
]
}
}
Delta
{
'channel': 'depth',
'timestamp': 1639083660401,
'data': {
'type': 'update',
'instrument_id': 'BTC-PERPETUAL',
'sequence': 1639042602148589842,
'prev_sequence': 1639042602148589841,
'changes': [
['sell', '47874.00000000', '0.00000000']
]
}
}
'''
if data['data']['type'] == 'update':
pair = self.exchange_symbol_to_std_symbol(
data['data'].get('instrument_id') or data['data'].get('pair'))
if data['data']['sequence'] != self._sequence_no[pair] + 1:
raise MissingSequenceNumber(
"Missing sequence number, restarting")
self._sequence_no[pair] = data['data']['sequence']
delta = {BID: [], ASK: []}
for side, price, amount in data['data']['changes']:
side = ASK if side == 'sell' else BID
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
delta[side].append((price, 0))
del self._l2_book[pair].book[side][price]
else:
delta[side].append((price, amount))
self._l2_book[pair].book[side][price] = amount
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(
data['timestamp']),
raw=data,
sequence_number=self._sequence_no[pair],
delta=delta)
else:
pair = self.exchange_symbol_to_std_symbol(
data['data'].get('instrument_id') or data['data'].get('pair'))
self._l2_book[pair] = OrderBook(
self.id,
pair,
max_depth=self.max_depth,
bids={
Decimal(price): Decimal(size)
for price, size in data['data']['bids']
},
asks={
Decimal(price): Decimal(size)
for price, size in data['data']['asks']
})
self._sequence_no[pair] = data['data']['sequence']
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(
data['timestamp']),
raw=data,
sequence_number=data['data']['sequence'])
async def _ticker(self, data: dict, timestamp: float):
'''
{
'channel': 'ticker',
'timestamp': 1639093870710,
'data': {
'time': 1639093870710,
'instrument_id': 'ETH-PERPETUAL',
'best_bid': '4155.85000000',
'best_ask': '4155.90000000',
'best_bid_qty': '2000.00000000',
'best_ask_qty': '3000.00000000',
'ask_sigma': '',
'bid_sigma': '',
'last_price': '4157.80000000',
'last_qty': '1000.00000000',
'open24h': '4436.75000000',
'high24h': '4490.00000000',
'low24h': '4086.60000000',
'price_change24h': '-0.06287260',
'volume24h': '1000218.00000000',
'open_interest': '7564685.00000000',
'funding_rate': '0.00025108',
'funding_rate8h': '0.00006396',
'mark_price': '4155.62874869',
'min_sell': '4030.50000000',
'max_buy': '4280.50000000'
}
}
'''
if data['data']['best_bid'] and data['data']['best_ask']:
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(
data['data'].get('instrument_id')
or data['data'].get('pair')),
Decimal(data['data']['best_bid']),
Decimal(data['data']['best_ask']),
self.timestamp_normalize(data['timestamp']),
raw=data)
await self.callback(TICKER, t, timestamp)
def _order_type_translate(self, t: str) -> str:
if t == 'limit':
return LIMIT
if t == 'market':
return MARKET
if t == 'stop-limit':
return STOP_LIMIT
if t == 'stop-market':
return STOP_MARKET
if t == 'trigger-limit':
return TRIGGER_LIMIT
if t == 'trigger-market':
return TRIGGER_MARKET
raise ValueError('Invalid order type detected %s', t)
def _status_translate(self, s: str) -> str:
if s == 'open':
return OPEN
if s == 'pending':
return PENDING
if s == 'filled':
return FILLED
if s == 'cancelled':
return CANCELLED
raise ValueError('Invalid order status detected %s', s)
async def _order(self, msg: dict, timestamp: float):
"""
{
"channel":"order",
"timestamp":1587994934089,
"data":[
{
"order_id":"1590",
"instrument_id":"BTC-1MAY20-8750-P",
"qty":"0.50000000",
"filled_qty":"0.10000000",
"remain_qty":"0.40000000",
"price":"0.16000000",
"avg_price":"0.16000000",
"side":"buy",
"order_type":"limit",
"time_in_force":"gtc",
"created_at":1587870609000,
"updated_at":1587870609000,
"status":"open",
"fee":"0.00002000",
"cash_flow":"-0.01600000",
"pnl":"0.00000000",
"is_liquidation": false,
"auto_price":"0.00000000",
"auto_price_type":"",
"taker_fee_rate": "0.00050000",
"maker_fee_rate": "0.00020000",
"label": "hedge",
"stop_price": "0.00000000",
"reduce_only": false,
"post_only": false,
"reject_post_only": false,
"mmp": false,
"reorder_index": 1
}
]
}
"""
for entry in msg['data']:
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(
entry['instrument_id']),
entry['order_id'],
BUY if entry['side'] == 'buy' else SELL,
self._status_translate(entry['status']),
self._order_type_translate(entry['order_type']),
Decimal(entry['price']),
Decimal(entry['filled_qty']),
Decimal(entry['remain_qty']),
self.timestamp_normalize(entry['updated_at']),
raw=entry)
await self.callback(ORDER_INFO, oi, timestamp)
async def _balances(self, msg: dict, timestamp: float):
'''
Futures/Options
{
"channel":"account",
"timestamp":1589031930115,
"data":{
"user_id":"53345",
"currency":"BTC",
"cash_balance":"9999.94981346",
"available_balance":"9999.90496213",
"margin_balance":"9999.94981421",
"initial_margin":"0.04485208",
"maintenance_margin":"0.00000114",
"equity":"10000.00074364",
"pnl":"0.00746583",
"total_delta":"0.06207078",
"account_id":"8",
"mode":"regular",
"session_upl":"0.00081244",
"session_rpl":"0.00000021",
"option_value":"0.05092943",
"option_pnl":"0.00737943",
"option_session_rpl":"0.00000000",
"option_session_upl":"0.00081190",
"option_delta":"0.11279249",
"option_gamma":"0.00002905",
"option_vega":"4.30272923",
"option_theta":"-3.08908220",
"future_pnl":"0.00008640",
"future_session_rpl":"0.00000021",
"future_session_upl":"0.00000054",
"future_session_funding":"0.00000021",
"future_delta":"0.00955630",
"created_at":1588997840512,
"projected_info": {
"projected_initial_margin": "0.97919888",
"projected_maintenance_margin": "0.78335911",
"projected_total_delta": "3.89635553"
}
}
}
Spot
{
'channel': 'account',
'timestamp': 1641516119102,
'data': {
'user_id': '979394',
'balances': [
{
'currency': 'BTC',
'available': '31.02527500',
'frozen': '0.00000000'
},
{
'currency': 'ETH',
'available': '110.00000000',
'frozen': '0.00000000'
}
]
}
}
'''
if 'balances' in msg['data']:
# Spot
for balance in msg['data']['balances']:
b = Balance(self.id,
balance['currency'],
Decimal(balance['available']),
Decimal(balance['frozen']),
raw=msg)
await self.callback(BALANCES, b, timestamp)
else:
b = Balance(self.id,
msg['data']['currency'],
Decimal(msg['data']['cash_balance']),
Decimal(msg['data']['cash_balance']) -
Decimal(msg['data']['available_balance']),
raw=msg)
await self.callback(BALANCES, b, timestamp)
async def _fill(self, msg: dict, timestamp: float):
'''
{
"channel":"user_trade",
"timestamp":1588997059737,
"data":[
{
"trade_id":2388418,
"order_id":1384232,
"instrument_id":"BTC-26JUN20-6000-P",
"qty":"0.10000000",
"price":"0.01800000",
"sigma":"1.15054346",
"underlying_price":"9905.54000000",
"index_price":"9850.47000000",
"usd_price":"177.30846000",
"fee":"0.00005000",
"fee_rate":"0.00050000",
"side":"buy",
"created_at":1588997060000,
"is_taker":true,
"order_type":"limit",
"is_block_trade":false,
"label": "hedge"
}
]
}
'''
for entry in msg['data']:
f = Fill(self.id,
self.exchange_symbol_to_std_symbol(
entry['instrument_id']),
BUY if entry['side'] == 'buy' else SELL,
Decimal(entry['qty']),
Decimal(entry['price']),
Decimal(entry['fee']),
str(entry['trade_id']),
str(entry['order_id']),
self._order_type_translate(entry['order_type']),
None,
self.timestamp_normalize(entry['created_at']),
raw=entry)
await self.callback(FILLS, f, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['channel'] == 'depth':
await self._book(msg, timestamp)
elif msg['channel'] == 'trade':
await self._trade(msg, timestamp)
elif msg['channel'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['channel'] == 'order':
await self._order(msg, timestamp)
elif msg['channel'] == 'account':
await self._balances(msg, timestamp)
elif msg['channel'] == 'user_trade':
await self._fill(msg, timestamp)
elif msg['channel'] == 'subscription':
"""
{
'channel': 'subscription',
'timestamp': 1639080572093,
'data': {'code': 0, 'subscription': ['trade']}
}
"""
if msg['data']['code'] == 0:
return
else:
LOG.warning(
"%s: error received from exchange while subscribing: %s",
self.id, msg)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class AscendEX(Feed):
id = ASCENDEX
websocket_endpoints = [WebsocketEndpoint('wss://ascendex.com/1/api/pro/v1/stream')]
rest_endpoints = [RestEndpoint('https://ascendex.com', routes=Routes('/api/pro/v1/products'))]
websocket_channels = {
L2_BOOK: 'depth:',
TRADES: 'trades:',
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data['data']:
# Only "Normal" status symbols are tradeable
if entry['status'] == 'Normal':
s = Symbol(entry['baseAsset'], entry['quoteAsset'])
ret[s.normalized] = entry['symbol']
info['tick_size'][s.normalized] = entry['tickSize']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
self.seq_no = defaultdict(lambda: None)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'm': 'trades',
'symbol': 'BTC/USDT',
'data': [{
'p': '23169.76',
'q': '0.00899',
'ts': 1608760026461,
'bm': False,
'seqnum': 72057614186183012
}]
}
"""
for trade in msg['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
SELL if trade['bm'] else BUY,
Decimal(trade['q']),
Decimal(trade['p']),
self.timestamp_normalize(trade['ts']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
sequence_number = msg['data']['seqnum']
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
delta = {BID: [], ASK: []}
if msg['m'] == 'depth-snapshot':
self.seq_no[pair] = sequence_number
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth)
else:
# ignore messages while we wait for the snapshot
if self.seq_no[pair] is None:
return
if self.seq_no[pair] + 1 != sequence_number:
raise MissingSequenceNumber
self.seq_no[pair] = sequence_number
for side in ('bids', 'asks'):
for price, amount in msg['data'][side]:
s = BID if side == 'bids' else ASK
price = Decimal(price)
size = Decimal(amount)
if size == 0:
delta[s].append((price, 0))
if price in self._l2_book[pair].book[s]:
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, size))
self._l2_book[pair].book[s][price] = size
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(msg['data']['ts']), raw=msg, delta=delta if msg['m'] != 'depth-snapshot' else None, sequence_number=sequence_number)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'm' in msg:
if msg['m'] == 'depth' or msg['m'] == 'depth-snapshot':
await self._book(msg, timestamp)
elif msg['m'] == 'trades':
await self._trade(msg, timestamp)
elif msg['m'] == 'ping':
await conn.write('{"op":"pong"}')
elif msg['m'] == 'connected':
return
elif msg['m'] == 'sub':
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
l2_pairs = []
for channel in self.subscription:
pairs = self.subscription[channel]
if channel == "depth:":
l2_pairs.extend(pairs)
message = {'op': 'sub', 'ch': channel + ','.join(pairs)}
await conn.write(json.dumps(message))
for pair in l2_pairs:
message = {"op": "req", "action": "depth-snapshot", "args": {"symbol": pair}}
await conn.write(json.dumps(message))
class dYdX(Feed, dYdXRestMixin):
id = DYDX
websocket_endpoints = [WebsocketEndpoint('wss://api.dydx.exchange/v3/ws')]
rest_endpoints = [
RestEndpoint('https://api.dydx.exchange', routes=Routes('/v3/markets'))
]
websocket_channels = {
L2_BOOK: 'v3_orderbook',
TRADES: 'v3_trades',
}
request_limit = 10
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for symbol, entry in data['markets'].items():
if entry['status'] != 'ONLINE':
continue
stype = entry['type'].lower()
s = Symbol(entry['baseAsset'], entry['quoteAsset'], type=stype)
ret[s.normalized] = symbol
info['tick_size'][s.normalized] = entry['tickSize']
info['instrument_type'][s.normalized] = stype
return ret, info
def __reset(self):
self._l2_book = {}
self._offsets = {}
async def _book(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['id'])
delta = {BID: [], ASK: []}
if msg['type'] == 'channel_data':
updated = False
offset = int(msg['contents']['offset'])
for side, key in ((BID, 'bids'), (ASK, 'asks')):
for data in msg['contents'][key]:
price = Decimal(data[0])
amount = Decimal(data[1])
if price in self._offsets[
pair] and offset < self._offsets[pair][price]:
continue
updated = True
self._offsets[pair][price] = offset
delta[side].append((price, amount))
if amount == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
else:
self._l2_book[pair].book[side][price] = amount
if updated:
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
delta=delta,
raw=msg)
else:
# snapshot
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._offsets[pair] = {}
for side, data in msg['contents'].items():
side = BID if side == 'bids' else ASK
for entry in data:
self._offsets[pair][Decimal(entry['price'])] = int(
entry['offset'])
size = Decimal(entry['size'])
if size > 0:
self._l2_book[pair].book[side][Decimal(
entry['price'])] = size
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
delta=None,
raw=msg)
async def _trade(self, msg: dict, timestamp: float):
"""
update:
{
'type': 'channel_data',
'connection_id': '7b4abf85-f9eb-4f6e-82c0-5479ad5681e9',
'message_id': 18,
'id': 'DOGE-USD',
'channel': 'v3_trades',
'contents': {
'trades': [{
'size': '390',
'side': 'SELL',
'price': '0.2334',
'createdAt': datetime.datetime(2021, 6, 23, 22, 36, 34, 520000, tzinfo=datetime.timezone.utc)
}]
}
}
initial message:
{
'type': 'subscribed',
'connection_id': 'ccd8b74c-97b3-491d-a9fc-4a92a171296e',
'message_id': 4,
'channel': 'v3_trades',
'id': 'UNI-USD',
'contents': {
'trades': [{
'side': 'BUY',
'size': '384.1',
'price': '17.23',
'createdAt': datetime.datetime(2021, 6, 23, 20, 28, 25, 465000, tzinfo=datetime.timezone.utc)
},
{
'side': 'SELL',
'size': '384.1',
'price': '17.138',
'createdAt': datetime.datetime(2021, 6, 23, 20, 22, 26, 466000, tzinfo=datetime.timezone.utc)},
}]
}
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['id'])
for trade in msg['contents']['trades']:
t = Trade(self.id,
pair,
BUY if trade['side'] == 'BUY' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
self.timestamp_normalize(trade['createdAt']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['type'] == 'channel_data' or msg['type'] == 'subscribed':
chan = self.exchange_channel_to_std(msg['channel'])
if chan == L2_BOOK:
await self._book(msg, timestamp)
elif chan == TRADES:
await self._trade(msg, timestamp)
else:
LOG.warning("%s: unexpected channel type received: %s",
self.id, msg)
elif msg['type'] == 'connected':
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan, symbols in self.subscription.items():
for symbol in symbols:
msg = {"type": "subscribe", "channel": chan, "id": symbol}
if self.exchange_channel_to_std(chan) == L2_BOOK:
msg['includeOffsets'] = True
await conn.write(json.dumps(msg))