class Bequant(Feed):
id = BEQUANT
rest_endpoints = [
RestEndpoint('https://api.bequant.io',
routes=Routes('/api/2/public/symbol'))
]
valid_candle_intervals = {
'1m', '3m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '1M'
}
candle_interval_map = {
'1m': 'M1',
'3m': 'M3',
'5m': 'M5',
'15m': 'M15',
'30m': 'M30',
'1h': 'H1',
'4h': 'H4',
'1d': 'D1',
'1w': 'D7',
'1M': '1M'
}
websocket_channels = {
BALANCES: 'subscribeBalance',
TRANSACTIONS: 'subscribeTransactions',
ORDER_INFO: 'subscribeReports',
L2_BOOK: 'subscribeOrderbook',
TRADES: 'subscribeTrades',
TICKER: 'subscribeTicker',
CANDLES: 'subscribeCandles'
}
websocket_endpoints = [
WebsocketEndpoint('wss://api.bequant.io/api/2/ws/public',
channel_filter=(websocket_channels[L2_BOOK],
websocket_channels[TRADES],
websocket_channels[TICKER],
websocket_channels[CANDLES])),
WebsocketEndpoint('wss://api.bequant.io/api/2/ws/trading',
channel_filter=(websocket_channels[ORDER_INFO], )),
WebsocketEndpoint('wss://api.bequant.io/api/2/ws/account',
channel_filter=(websocket_channels[BALANCES],
websocket_channels[TRANSACTIONS])),
]
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
normalized_currencies = {
'USD': 'USDT',
'USDB': 'USD',
}
for symbol in data:
# Filter out pairs ending in _BQX
# From Bequant support: "BQX pairs are our 0 taker fee pairs that are only to be used by our retail broker clients (the BQX is to differentiate them from the traditional pairs)"
if symbol['id'][-4:] == '_BQX':
continue
base_currency = normalized_currencies[
symbol['baseCurrency']] if symbol[
'baseCurrency'] in normalized_currencies else symbol[
'baseCurrency']
quote_currency = normalized_currencies[
symbol['quoteCurrency']] if symbol[
'quoteCurrency'] in normalized_currencies else symbol[
'quoteCurrency']
s = Symbol(base_currency, quote_currency)
ret[s.normalized] = symbol['id']
info['tick_size'][s.normalized] = symbol['tickSize']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
self.seq_no = {}
async def _ticker(self, msg: dict, timestamp: float):
"""
{
"ask": "0.054464", <- best ask
"bid": "0.054463", <- best bid
"last": "0.054463", <- last trade
"open": "0.057133", <- last trade price 24hrs previous
"low": "0.053615", <- lowest trade in past 24hrs
"high": "0.057559", <- highest trade in past 24hrs
"volume": "33068.346", <- total base currency traded in past 24hrs
"volumeQuote": "1832.687530809", <- total quote currency traded in past 24hrs
"timestamp": "2017-10-19T15:45:44.941Z", <- last update or refresh ticker timestamp
"symbol": "ETHBTC"
}
"""
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
Decimal(msg['bid']),
Decimal(msg['ask']),
self.timestamp_normalize(msg['timestamp']),
raw=msg)
await self.callback(TICKER, t, timestamp)
async def _book_snapshot(self, msg: dict, ts: float):
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
self._l2_book[pair] = OrderBook(self.id,
pair,
bids={
Decimal(bid['price']):
Decimal(bid['size'])
for bid in msg['bid']
},
asks={
Decimal(ask['price']):
Decimal(ask['size'])
for ask in msg['ask']
})
await self.book_callback(L2_BOOK,
self._l2_book[pair],
ts,
raw=msg,
timestamp=self.timestamp_normalize(
msg['timestamp']))
async def _book_update(self, msg: dict, ts: float):
delta = {BID: [], ASK: []}
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
for side in ('bid', 'ask'):
s = BID if side == 'bid' else ASK
for entry in msg[side]:
price = Decimal(entry['price'])
amount = Decimal(entry['size'])
if amount == 0:
delta[s].append((price, 0))
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, amount))
self._l2_book[pair].book[s][price] = amount
await self.book_callback(L2_BOOK,
self._l2_book[pair],
ts,
timestamp=self.timestamp_normalize(
msg['timestamp']),
raw=msg,
sequence_number=self.seq_no[msg['symbol']],
delta=delta)
async def _trades(self, msg: dict, timestamp: float):
"""
"params": {
"data": [
{
"id": 54469813,
"price": "0.054670",
"quantity": "0.183",
"side": "buy",
"timestamp": "2017-10-19T16:34:25.041Z"
}
],
"symbol": "ETHBTC"
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
for update in msg['data']:
t = Trade(self.id,
pair,
BUY if update['side'] == 'buy' else SELL,
Decimal(update['quantity']),
Decimal(update['price']),
self.timestamp_normalize(update['timestamp']),
id=str(update['id']),
raw=update)
await self.callback(TRADES, t, timestamp)
async def _candles(self, msg: dict, timestamp: float):
"""
{
"jsonrpc": "2.0",
"method": "updateCandles",
"params": {
"data": [
{
"timestamp": "2017-10-19T16:30:00.000Z",
"open": "0.054614",
"close": "0.054465",
"min": "0.054339",
"max": "0.054724",
"volume": "141.268",
"volumeQuote": "7.709353873"
}
],
"symbol": "ETHBTC",
"period": "M30"
}
}
"""
interval = str(self.normalize_candle_interval[msg['period']])
for candle in msg['data']:
start = self.timestamp_normalize(candle['timestamp'])
end = start + timedelta_str_to_sec(interval) - 1
c = Candle(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
start,
end,
interval,
None,
Decimal(candle['open']),
Decimal(candle['close']),
Decimal(candle['max']),
Decimal(candle['min']),
Decimal(candle['volume']),
None,
None,
raw=candle)
await self.callback(CANDLES, c, timestamp)
async def _order_status(self, msg: str, ts: float):
status_lookup = {
'new': OPEN,
'partiallyFilled': PARTIAL,
'filled': FILLED,
'canceled': CANCELLED,
'expired': EXPIRED,
'suspended': SUSPENDED,
}
type_lookup = {
'limit': LIMIT,
'market': MARKET,
'stopLimit': STOP_LIMIT,
'stopMarket': STOP_MARKET,
}
"""
Example response:
{
"jsonrpc": "2.0",
"method": "report",
"params": {
"id": "4345697765",
"clientOrderId": "53b7cf917963464a811a4af426102c19",
"symbol": "ETHBTC",
"side": "sell",
"status": "filled",
"type": "limit",
"timeInForce": "GTC",
"quantity": "0.001",
"price": "0.053868",
"cumQuantity": "0.001",
"postOnly": false,
"createdAt": "2017-10-20T12:20:05.952Z",
"updatedAt": "2017-10-20T12:20:38.708Z",
"reportType": "trade",
"tradeQuantity": "0.001",
"tradePrice": "0.053868",
"tradeId": 55051694,
"tradeFee": "-0.000000005"
}
}
"""
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(msg["symbol"]),
msg["id"],
SELL if msg["side"] == 'sell' else BUY,
status_lookup[msg["status"]],
type_lookup[msg["type"]],
Decimal(msg['price']),
Decimal(msg['cumQuantity']),
Decimal(msg['quantity']) - Decimal(msg['cumQuantity']),
self.timestamp_normalize(msg["updatedAt"]) if msg["updatedAt"] else
self.timestamp_normalize(msg["createdAt"]),
raw=msg)
await self.callback(ORDER_INFO, oi, ts)
async def _transactions(self, msg: str, ts: float):
"""
A transaction notification occurs each time the transaction has been changed, such as creating a transaction,
updating the pending state (for example the hash assigned) or completing a transaction.
This is the easiest way to track deposits or develop real-time asset monitoring.
{
"jsonrpc": "2.0",
"method": "updateTransaction",
"params": {
"id": "76b70d1c-3dd7-423e-976e-902e516aae0e",
"index": 7173627250,
"type": "bankToExchange",
"status": "success",
"currency": "BTG",
"amount": "0.00001000",
"createdAt": "2021-01-31T08:19:33.892Z",
"updatedAt": "2021-01-31T08:19:33.967Z"
}
}
"""
t = Transaction(self.id,
msg['params']['currency'],
msg['params']['type'],
msg['params']['status'],
Decimal(msg['params']['amount']),
msg['params']['createdAt'].timestamp(),
raw=msg)
await self.callback(TRANSACTIONS, t, ts)
async def _balances(self, msg: str, ts: float):
'''
{
"jsonrpc": "2.0",
"method": "balance",
"params": [
{
"currency": "BTC",
"available": "0.00005821",
"reserved": "0"
},
{
"currency": "DOGE",
"available": "11",
"reserved": "0"
}
]
}
'''
for entry in msg['params']:
b = Balance(self.id,
entry['currency'],
Decimal(entry['available']),
Decimal(entry['reserved']),
raw=entry)
await self.callback(BALANCES, b, ts)
async def message_handler(self, msg: str, conn: AsyncConnection,
ts: float):
msg = json.loads(msg, parse_float=Decimal)
if 'params' in msg and 'sequence' in msg['params']:
pair = msg['params']['symbol']
if pair in self.seq_no:
if self.seq_no[pair] + 1 != msg['params']['sequence']:
if self.seq_no[pair] >= msg['params']['sequence']:
return
LOG.warning("%s: Missing sequence number detected for %s",
self.id, pair)
raise MissingSequenceNumber(
"Missing sequence number, restarting")
self.seq_no[pair] = msg['params']['sequence']
if 'method' in msg:
m = msg['method']
params = msg['params']
if m == 'ticker':
await self._ticker(params, ts)
elif m == 'snapshotOrderbook':
await self._book_snapshot(params, ts)
elif m == 'updateOrderbook':
await self._book_update(params, ts)
elif m in ('updateTrades', 'snapshotTrades'):
await self._trades(params, ts)
elif m in ('snapshotCandles', 'updateCandles'):
await self._candles(params, ts)
elif m in ('activeOrders', 'report'):
if isinstance(params, list):
for entry in params:
await self._order_status(entry, ts)
else:
await self._order_status(params, ts)
elif m == 'updateTransaction':
await self._transactions(params, ts)
elif m == 'balance':
await self._balances(msg, conn, ts)
else:
LOG.warning(
f"{self.id}: Invalid message received on {conn.uuid}: {msg}"
)
else:
if 'error' in msg:
LOG.error(
f"{self.id}: Received error on {conn.uuid}: {msg['error']}"
)
async def authenticate(self, conn: AsyncConnection):
if self.requires_authentication:
# https://api.bequant.io/#socket-session-authentication
# Nonce should be random string
nonce = 'h'.join(
random.choices(string.ascii_letters + string.digits,
k=16)).encode('utf-8')
signature = hmac.new(self.key_secret.encode('utf-8'), nonce,
hashlib.sha256).hexdigest()
auth = {
"method": "login",
"params": {
"algo": "HS256",
"pKey": self.key_id,
"nonce": nonce.decode(),
"signature": signature
},
"id": conn.uuid
}
await conn.write(json.dumps(auth))
LOG.debug(f"{conn.uuid}: Authenticating with message: {auth}")
return conn
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan, symbols in conn.subscription.items():
# These channel subs fail if provided with symbol data. "params" must be blank.
if chan in [
'subscribeTransactions', 'subscribeBalance',
'subscribeReports'
]:
LOG.debug(f'Subscribing to {chan} with no symbols')
await conn.write(
json.dumps({
"method": chan,
"params": {},
"id": conn.uuid
}))
else:
for symbol in symbols:
params = {
"symbol": symbol,
}
if chan == "subscribeCandles":
params['period'] = self.candle_interval_map[
self.candle_interval]
LOG.debug(
f'{self.id}: Subscribing to "{chan}" with params {params}'
)
await conn.write(
json.dumps({
"method": chan,
"params": params,
"id": conn.uuid
}))
class Bitmex(Feed, BitmexRestMixin):
id = BITMEX
websocket_endpoints = [WebsocketEndpoint('wss://www.bitmex.com/realtime', sandbox='wss://testnet.bitmex.com/realtime', options={'compression': None})]
rest_endpoints = [RestEndpoint('https://www.bitmex.com', routes=Routes('/api/v1/instrument/active'))]
websocket_channels = {
L2_BOOK: 'orderBookL2',
TRADES: 'trade',
TICKER: 'quote',
FUNDING: 'funding',
ORDER_INFO: 'order',
OPEN_INTEREST: 'instrument',
LIQUIDATIONS: 'liquidation'
}
request_limit = 0.5
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
base = entry['rootSymbol'].replace("XBT", "BTC")
quote = entry['quoteCurrency'].replace("XBT", "BTC")
stype = PERPETUAL
if entry['expiry']:
stype = FUTURES
s = Symbol(base, quote, type=stype, expiry_date=entry['expiry'])
ret[s.normalized] = entry['symbol']
info['tick_size'][s.normalized] = entry['tickSize']
info['instrument_type'][s.normalized] = stype
return ret, info
def _reset(self):
self.partial_received = defaultdict(bool)
self.order_id = {}
self.open_orders = {}
for pair in self.normalized_symbols:
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth)
self.order_id[pair] = defaultdict(dict)
@staticmethod
def normalize_order_status(status):
status_map = {
'New': OPEN,
'Filled': FILLED,
'Canceled': CANCELLED,
}
return status_map[status]
def init_order_info(self, o):
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(o['symbol']),
o['orderID'],
BUY if o['side'] == 'Buy' else SELL,
self.normalize_order_status(o['ordStatus']),
LIMIT if o['ordType'].lower() == 'limit' else MARKET if o['ordType'].lower() == 'market' else None,
Decimal(o['avgPx']) if o['avgPx'] else Decimal(o['price']),
Decimal(o['orderQty']),
Decimal(o['leavesQty']),
self.timestamp_normalize(o['timestamp']),
raw=str(o), # Need to convert to string to avoid json serialization error when updating order
)
return oi
async def _order(self, msg: dict, timestamp: float):
"""
order msg example
{
"table": "order",
"action": "partial",
"keys": [
"orderID"
],
"types": {
"orderID": "guid",
"clOrdID": "string",
"clOrdLinkID": "symbol",
"account": "long",
"symbol": "symbol",
"side": "symbol",
"simpleOrderQty": "float",
"orderQty": "long",
"price": "float",
"displayQty": "long",
"stopPx": "float",
"pegOffsetValue": "float",
"pegPriceType": "symbol",
"currency": "symbol",
"settlCurrency": "symbol",
"ordType": "symbol",
"timeInForce": "symbol",
"execInst": "symbol",
"contingencyType": "symbol",
"exDestination": "symbol",
"ordStatus": "symbol",
"triggered": "symbol",
"workingIndicator": "boolean",
"ordRejReason": "symbol",
"simpleLeavesQty": "float",
"leavesQty": "long",
"simpleCumQty": "float",
"cumQty": "long",
"avgPx": "float",
"multiLegReportingType": "symbol",
"text": "string",
"transactTime": "timestamp",
"timestamp": "timestamp"
},
"foreignKeys": {
"symbol": "instrument",
"side": "side",
"ordStatus": "ordStatus"
},
"attributes": {
"orderID": "grouped",
"account": "grouped",
"ordStatus": "grouped",
"workingIndicator": "grouped"
},
"filter": {
"account": 1600000,
"symbol": "ETHUSDTH22"
},
"data": [
{
"orderID": "360fad5a-49e3-4187-ad04-8fac82b8a95f",
"clOrdID": "",
"clOrdLinkID": "",
"account": 1600000,
"symbol": "ETHUSDTH22",
"side": "Buy",
"simpleOrderQty": null,
"orderQty": 1000,
"price": 2000,
"displayQty": null,
"stopPx": null,
"pegOffsetValue": null,
"pegPriceType": "",
"currency": "USDT",
"settlCurrency": "USDt",
"ordType": "Limit",
"timeInForce": "GoodTillCancel",
"execInst": "",
"contingencyType": "",
"exDestination": "XBME",
"ordStatus": "New",
"triggered": "",
"workingIndicator": true,
"ordRejReason": "",
"simpleLeavesQty": null,
"leavesQty": 1000,
"simpleCumQty": null,
"cumQty": 0,
"avgPx": null,
"multiLegReportingType": "SingleSecurity",
"text": "Submitted via API.",
"transactTime": "2022-02-13T00:15:02.570000Z",
"timestamp": "2022-02-13T00:15:02.570000Z"
},
{
"orderID": "74d2ad0a-49f1-44dc-820f-5f0cfd64c1a3",
"clOrdID": "",
"clOrdLinkID": "",
"account": 1600000,
"symbol": "ETHUSDTH22",
"side": "Buy",
"simpleOrderQty": null,
"orderQty": 1000,
"price": 2000,
"displayQty": null,
"stopPx": null,
"pegOffsetValue": null,
"pegPriceType": "",
"currency": "USDT",
"settlCurrency": "USDt",
"ordType": "Limit",
"timeInForce": "GoodTillCancel",
"execInst": "",
"contingencyType": "",
"exDestination": "XBME",
"ordStatus": "New",
"triggered": "",
"workingIndicator": true,
"ordRejReason": "",
"simpleLeavesQty": null,
"leavesQty": 1000,
"simpleCumQty": null,
"cumQty": 0,
"avgPx": null,
"multiLegReportingType": "SingleSecurity",
"text": "Submitted via API.",
"transactTime": "2022-02-13T00:17:13.796000Z",
"timestamp": "2022-02-13T00:17:13.796000Z"
}
]
}
{
"table": "order",
"action": "insert",
"data": [
{
"orderID": "0c4e4a8e-b234-495f-8b94-c4766786c4a5",
"clOrdID": "",
"clOrdLinkID": "",
"account": 1600000,
"symbol": "ETHUSDTH22",
"side": "Buy",
"simpleOrderQty": null,
"orderQty": 1000,
"price": 2000,
"displayQty": null,
"stopPx": null,
"pegOffsetValue": null,
"pegPriceType": "",
"currency": "USDT",
"settlCurrency": "USDt",
"ordType": "Limit",
"timeInForce": "GoodTillCancel",
"execInst": "",
"contingencyType": "",
"exDestination": "XBME",
"ordStatus": "New",
"triggered": "",
"workingIndicator": true,
"ordRejReason": "",
"simpleLeavesQty": null,
"leavesQty": 1000,
"simpleCumQty": null,
"cumQty": 0,
"avgPx": null,
"multiLegReportingType": "SingleSecurity",
"text": "Submitted via API.",
"transactTime": "2022-02-13T00:21:50.268000Z",
"timestamp": "2022-02-13T00:21:50.268000Z"
}
]
}
{
"table": "order",
"action": "update",
"data": [
{
"orderID": "360fa95a-49e3-4187-ad04-8fac82b8a95f",
"ordStatus": "Canceled",
"workingIndicator": false,
"leavesQty": 0,
"text": "Canceled: Cancel from www.bitmex.com\nSubmitted via API.",
"timestamp": "2022-02-13T08:16:36.446000Z",
"clOrdID": "",
"account": 1600000,
"symbol": "ETHUSDTH22"
}
]
}
"""
if msg['action'] == 'partial':
# Initial snapshot of open orders
self.open_orders = {}
for o in msg['data']:
oi = self.init_order_info(o)
self.open_orders[oi.id] = oi
elif msg['action'] == 'insert':
for o in msg['data']:
oi = self.init_order_info(o)
self.open_orders[oi.id] = oi
await self.callback(ORDER_INFO, oi, timestamp)
elif msg['action'] == 'update':
for o in msg['data']:
oi = self.open_orders.get(o['orderID'])
if oi:
info = oi.to_dict()
if 'ordStatus' in o:
info['status'] = self.normalize_order_status(o['ordStatus'])
if 'leaveQty' in o:
info['remaining'] = Decimal(o['leavesQty'])
if 'avgPx' in o:
info['price'] = Decimal(o['avgPx'])
info['raw'] = str(o) # Not sure if this is needed
new_oi = OrderInfo(**info)
if new_oi.status in (FILLED, CANCELLED):
self.open_orders.pop(new_oi.id)
else:
self.open_orders[new_oi.id] = oi
await self.callback(ORDER_INFO, new_oi, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def _trade(self, msg: dict, timestamp: float):
"""
trade msg example
{
'timestamp': '2018-05-19T12:25:26.632Z',
'symbol': 'XBTUSD',
'side': 'Buy',
'size': 40,
'price': 8335,
'tickDirection': 'PlusTick',
'trdMatchID': '5f4ecd49-f87f-41c0-06e3-4a9405b9cdde',
'grossValue': 479920,
'homeNotional': Decimal('0.0047992'),
'foreignNotional': 40
}
"""
for data in msg['data']:
ts = self.timestamp_normalize(data['timestamp'])
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
BUY if data['side'] == 'Buy' else SELL,
Decimal(data['size']),
Decimal(data['price']),
ts,
id=data['trdMatchID'],
raw=data
)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
"""
the Full bitmex book
"""
# PERF perf_start(self.id, 'book_msg')
if not msg['data']:
# see https://github.com/bmoscon/cryptofeed/issues/688
# msg['data'] can be an empty list
return
delta = None
# if we reset the book, force a full update
pair = self.exchange_symbol_to_std_symbol(msg['data'][0]['symbol'])
if not self.partial_received[pair]:
# per bitmex documentation messages received before partial
# should be discarded
if msg['action'] != 'partial':
return
self.partial_received[pair] = True
if msg['action'] == 'partial':
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
price = Decimal(data['price'])
size = Decimal(data['size'])
order_id = data['id']
self._l2_book[pair].book[side][price] = size
self.order_id[pair][side][order_id] = price
elif msg['action'] == 'insert':
delta = {BID: [], ASK: []}
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
price = Decimal(data['price'])
size = Decimal(data['size'])
order_id = data['id']
self._l2_book[pair].book[side][price] = size
self.order_id[pair][side][order_id] = price
delta[side].append((price, size))
elif msg['action'] == 'update':
delta = {BID: [], ASK: []}
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
update_size = Decimal(data['size'])
order_id = data['id']
price = self.order_id[pair][side][order_id]
self._l2_book[pair].book[side][price] = update_size
self.order_id[pair][side][order_id] = price
delta[side].append((price, update_size))
elif msg['action'] == 'delete':
delta = {BID: [], ASK: []}
for data in msg['data']:
side = BID if data['side'] == 'Buy' else ASK
order_id = data['id']
delete_price = self.order_id[pair][side][order_id]
del self.order_id[pair][side][order_id]
del self._l2_book[pair].book[side][delete_price]
delta[side].append((delete_price, 0))
else:
LOG.warning("%s: Unexpected l2 Book message %s", self.id, msg)
return
# PERF perf_end(self.id, 'book_msg')
# PERF perf_log(self.id, 'book_msg')
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg, delta=delta)
async def _ticker(self, msg: dict, timestamp: float):
for data in msg['data']:
t = Ticker(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
Decimal(data['bidPrice']),
Decimal(data['askPrice']),
self.timestamp_normalize(data['timestamp']),
raw=data
)
await self.callback(TICKER, t, timestamp)
async def _funding(self, msg: dict, timestamp: float):
"""
{'table': 'funding',
'action': 'partial',
'keys': ['timestamp', 'symbol'],
'types': {
'timestamp': 'timestamp',
'symbol': 'symbol',
'fundingInterval': 'timespan',
'fundingRate': 'float',
'fundingRateDaily': 'float'
},
'foreignKeys': {
'symbol': 'instrument'
},
'attributes': {
'timestamp': 'sorted',
'symbol': 'grouped'
},
'filter': {'symbol': 'XBTUSD'},
'data': [{
'timestamp': '2018-08-21T20:00:00.000Z',
'symbol': 'XBTUSD',
'fundingInterval': '2000-01-01T08:00:00.000Z',
'fundingRate': Decimal('-0.000561'),
'fundingRateDaily': Decimal('-0.001683')
}]
}
"""
for data in msg['data']:
ts = self.timestamp_normalize(data['timestamp'])
interval = data['fundingInterval']
f = Funding(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
None,
data['fundingRate'],
self.timestamp_normalize(data['timestamp'] + timedelta(hours=interval.hour)),
ts,
raw=data
)
await self.callback(FUNDING, f, timestamp)
async def _instrument(self, msg: dict, timestamp: float):
"""
Example instrument data
{
'table':'instrument',
'action':'partial',
'keys':[
'symbol'
],
'types':{
'symbol':'symbol',
'rootSymbol':'symbol',
'state':'symbol',
'typ':'symbol',
'listing':'timestamp',
'front':'timestamp',
'expiry':'timestamp',
'settle':'timestamp',
'relistInterval':'timespan',
'inverseLeg':'symbol',
'sellLeg':'symbol',
'buyLeg':'symbol',
'optionStrikePcnt':'float',
'optionStrikeRound':'float',
'optionStrikePrice':'float',
'optionMultiplier':'float',
'positionCurrency':'symbol',
'underlying':'symbol',
'quoteCurrency':'symbol',
'underlyingSymbol':'symbol',
'reference':'symbol',
'referenceSymbol':'symbol',
'calcInterval':'timespan',
'publishInterval':'timespan',
'publishTime':'timespan',
'maxOrderQty':'long',
'maxPrice':'float',
'lotSize':'long',
'tickSize':'float',
'multiplier':'long',
'settlCurrency':'symbol',
'underlyingToPositionMultiplier':'long',
'underlyingToSettleMultiplier':'long',
'quoteToSettleMultiplier':'long',
'isQuanto':'boolean',
'isInverse':'boolean',
'initMargin':'float',
'maintMargin':'float',
'riskLimit':'long',
'riskStep':'long',
'limit':'float',
'capped':'boolean',
'taxed':'boolean',
'deleverage':'boolean',
'makerFee':'float',
'takerFee':'float',
'settlementFee':'float',
'insuranceFee':'float',
'fundingBaseSymbol':'symbol',
'fundingQuoteSymbol':'symbol',
'fundingPremiumSymbol':'symbol',
'fundingTimestamp':'timestamp',
'fundingInterval':'timespan',
'fundingRate':'float',
'indicativeFundingRate':'float',
'rebalanceTimestamp':'timestamp',
'rebalanceInterval':'timespan',
'openingTimestamp':'timestamp',
'closingTimestamp':'timestamp',
'sessionInterval':'timespan',
'prevClosePrice':'float',
'limitDownPrice':'float',
'limitUpPrice':'float',
'bankruptLimitDownPrice':'float',
'bankruptLimitUpPrice':'float',
'prevTotalVolume':'long',
'totalVolume':'long',
'volume':'long',
'volume24h':'long',
'prevTotalTurnover':'long',
'totalTurnover':'long',
'turnover':'long',
'turnover24h':'long',
'homeNotional24h':'float',
'foreignNotional24h':'float',
'prevPrice24h':'float',
'vwap':'float',
'highPrice':'float',
'lowPrice':'float',
'lastPrice':'float',
'lastPriceProtected':'float',
'lastTickDirection':'symbol',
'lastChangePcnt':'float',
'bidPrice':'float',
'midPrice':'float',
'askPrice':'float',
'impactBidPrice':'float',
'impactMidPrice':'float',
'impactAskPrice':'float',
'hasLiquidity':'boolean',
'openInterest':'long',
'openValue':'long',
'fairMethod':'symbol',
'fairBasisRate':'float',
'fairBasis':'float',
'fairPrice':'float',
'markMethod':'symbol',
'markPrice':'float',
'indicativeTaxRate':'float',
'indicativeSettlePrice':'float',
'optionUnderlyingPrice':'float',
'settledPrice':'float',
'timestamp':'timestamp'
},
'foreignKeys':{
'inverseLeg':'instrument',
'sellLeg':'instrument',
'buyLeg':'instrument'
},
'attributes':{
'symbol':'unique'
},
'filter':{
'symbol':'XBTUSD'
},
'data':[
{
'symbol':'XBTUSD',
'rootSymbol':'XBT',
'state':'Open',
'typ':'FFWCSX',
'listing':'2016-05-13T12:00:00.000Z',
'front':'2016-05-13T12:00:00.000Z',
'expiry':None,
'settle':None,
'relistInterval':None,
'inverseLeg':'',
'sellLeg':'',
'buyLeg':'',
'optionStrikePcnt':None,
'optionStrikeRound':None,
'optionStrikePrice':None,
'optionMultiplier':None,
'positionCurrency':'USD',
'underlying':'XBT',
'quoteCurrency':'USD',
'underlyingSymbol':'XBT=',
'reference':'BMEX',
'referenceSymbol':'.BXBT',
'calcInterval':None,
'publishInterval':None,
'publishTime':None,
'maxOrderQty':10000000,
'maxPrice':1000000,
'lotSize':1,
'tickSize':Decimal( '0.5' ),
'multiplier':-100000000,
'settlCurrency':'XBt',
'underlyingToPositionMultiplier':None,
'underlyingToSettleMultiplier':-100000000,
'quoteToSettleMultiplier':None,
'isQuanto':False,
'isInverse':True,
'initMargin':Decimal( '0.01' ),
'maintMargin':Decimal( '0.005' ),
'riskLimit':20000000000,
'riskStep':10000000000,
'limit':None,
'capped':False,
'taxed':True,
'deleverage':True,
'makerFee':Decimal( '-0.00025' ),
'takerFee':Decimal( '0.00075' ),
'settlementFee':0,
'insuranceFee':0,
'fundingBaseSymbol':'.XBTBON8H',
'fundingQuoteSymbol':'.USDBON8H',
'fundingPremiumSymbol':'.XBTUSDPI8H',
'fundingTimestamp':'2020-02-02T04:00:00.000Z',
'fundingInterval':'2000-01-01T08:00:00.000Z',
'fundingRate':Decimal( '0.000106' ),
'indicativeFundingRate':Decimal( '0.0001' ),
'rebalanceTimestamp':None,
'rebalanceInterval':None,
'openingTimestamp':'2020-02-02T00:00:00.000Z',
'closingTimestamp':'2020-02-02T01:00:00.000Z',
'sessionInterval':'2000-01-01T01:00:00.000Z',
'prevClosePrice':Decimal( '9340.63' ),
'limitDownPrice':None,
'limitUpPrice':None,
'bankruptLimitDownPrice':None,
'bankruptLimitUpPrice':None,
'prevTotalVolume':1999389257669,
'totalVolume':1999420432348,
'volume':31174679,
'volume24h':1605909209,
'prevTotalTurnover':27967114248663460,
'totalTurnover':27967447182062520,
'turnover':332933399058,
'turnover24h':17126993087717,
'homeNotional24h':Decimal( '171269.9308771703' ),
'foreignNotional24h':1605909209,
'prevPrice24h':9348,
'vwap':Decimal( '9377.3443' ),
'highPrice':9464,
'lowPrice':Decimal( '9287.5' ),
'lastPrice':9352,
'lastPriceProtected':9352,
'lastTickDirection':'ZeroMinusTick',
'lastChangePcnt':Decimal( '0.0004' ),
'bidPrice':9352,
'midPrice':Decimal( '9352.25' ),
'askPrice':Decimal( '9352.5' ),
'impactBidPrice':Decimal( '9351.9125' ),
'impactMidPrice':Decimal( '9352.25' ),
'impactAskPrice':Decimal( '9352.7871' ),
'hasLiquidity':True,
'openInterest':983043322,
'openValue':10518563545400,
'fairMethod':'FundingRate',
'fairBasisRate':Decimal( '0.11607' ),
'fairBasis':Decimal( '0.43' ),
'fairPrice':Decimal( '9345.36' ),
'markMethod':'FairPrice',
'markPrice':Decimal( '9345.36' ),
'indicativeTaxRate':0,
'indicativeSettlePrice':Decimal( '9344.93' ),
'optionUnderlyingPrice':None,
'settledPrice':None,
'timestamp':'2020-02-02T00:30:43.772Z'
}
]
}
"""
for data in msg['data']:
if 'openInterest' in data:
ts = self.timestamp_normalize(data['timestamp'])
oi = OpenInterest(self.id, self.exchange_symbol_to_std_symbol(data['symbol']), Decimal(data['openInterest']), ts, raw=data)
await self.callback(OPEN_INTEREST, oi, timestamp)
async def _liquidation(self, msg: dict, timestamp: float):
"""
liquidation msg example
{
'orderID': '9513c849-ca0d-4e11-8190-9d221972288c',
'symbol': 'XBTUSD',
'side': 'Buy',
'price': 6833.5,
'leavesQty': 2020
}
"""
if msg['action'] == 'insert':
for data in msg['data']:
liq = Liquidation(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
BUY if data['side'] == 'Buy' else SELL,
Decimal(data['leavesQty']),
Decimal(data['price']),
data['orderID'],
UNFILLED,
None,
raw=data
)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'table' in msg:
if msg['table'] == 'trade':
await self._trade(msg, timestamp)
elif msg['table'] == 'order':
await self._order(msg, timestamp)
elif msg['table'] == 'orderBookL2':
await self._book(msg, timestamp)
elif msg['table'] == 'funding':
await self._funding(msg, timestamp)
elif msg['table'] == 'instrument':
await self._instrument(msg, timestamp)
elif msg['table'] == 'quote':
await self._ticker(msg, timestamp)
elif msg['table'] == 'liquidation':
await self._liquidation(msg, timestamp)
else:
LOG.warning("%s: Unhandled table=%r in %r", conn.uuid, msg['table'], msg)
elif 'info' in msg:
LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg)
elif 'subscribe' in msg:
if not msg['success']:
LOG.error("%s: Subscribe failure: %s", conn.uuid, msg)
elif 'error' in msg:
LOG.error("%s: Error message from exchange: %s", conn.uuid, msg)
elif 'request' in msg:
if msg['success']:
LOG.debug("%s: Success %s", conn.uuid, msg['request'].get('op'))
else:
LOG.warning("%s: Failure %s", conn.uuid, msg['request'])
else:
LOG.warning("%s: Unexpected message from exchange: %s", conn.uuid, msg)
async def subscribe(self, conn: AsyncConnection):
self._reset()
await self._authenticate(conn)
chans = []
for chan in self.subscription:
for pair in self.subscription[chan]:
chans.append(f"{chan}:{pair}")
for i in range(0, len(chans), 10):
await conn.write(json.dumps({"op": "subscribe",
"args": chans[i:i + 10]}))
async def _authenticate(self, conn: AsyncConnection):
"""Send API Key with signed message."""
# Docs: https://www.bitmex.com/app/apiKeys
# https://github.com/BitMEX/sample-market-maker/blob/master/test/websocket-apikey-auth-test.py
if self.key_id and self.key_secret:
LOG.info('%s: Authenticate with signature', conn.uuid)
expires = int(time.time()) + 365 * 24 * 3600 # One year
msg = f'GET/realtime{expires}'.encode('utf-8')
signature = hmac.new(self.key_secret.encode('utf-8'), msg, digestmod=hashlib.sha256).hexdigest()
await conn.write(json.dumps({'op': 'authKeyExpires', 'args': [self.key_id, expires, signature]}))
class Coinbase(Feed, CoinbaseRestMixin):
id = COINBASE
websocket_endpoints = [WebsocketEndpoint('wss://ws-feed.pro.coinbase.com', options={'compression': None})]
rest_endpoints = [RestEndpoint('https://api.pro.coinbase.com', routes=Routes('/products', l3book='/products/{}/book?level=3'))]
websocket_channels = {
L2_BOOK: 'level2',
L3_BOOK: 'full',
TRADES: 'matches',
TICKER: 'ticker',
}
request_limit = 10
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
sym = Symbol(entry['base_currency'], entry['quote_currency'])
info['tick_size'][sym.normalized] = entry['quote_increment']
info['instrument_type'][sym.normalized] = sym.type
ret[sym.normalized] = entry['id']
return ret, info
def __init__(self, callbacks=None, **kwargs):
super().__init__(callbacks=callbacks, **kwargs)
# we only keep track of the L3 order book if we have at least one subscribed order-book callback.
# use case: subscribing to the L3 book plus Trade type gives you order_type information (see _received below),
# and we don't need to do the rest of the book-keeping unless we have an active callback
self.keep_l3_book = False
if callbacks and L3_BOOK in callbacks:
self.keep_l3_book = True
self.__reset()
def __reset(self):
self.order_map = {}
self.order_type_map = {}
self.seq_no = None
# sequence number validation only works when the FULL data stream is enabled
chan = self.std_channel_to_exchange(L3_BOOK)
if chan in self.subscription:
pairs = self.subscription[chan]
self.seq_no = {pair: None for pair in pairs}
self._l3_book = {}
self._l2_book = {}
async def _ticker(self, msg: dict, timestamp: float):
'''
{
'type': 'ticker',
'sequence': 5928281084,
'product_id': 'BTC-USD',
'price': '8500.01000000',
'open_24h': '8217.24000000',
'volume_24h': '4529.1293778',
'low_24h': '8172.00000000',
'high_24h': '8600.00000000',
'volume_30d': '329178.93594133',
'best_bid': '8500',
'best_ask': '8500.01'
}
{
'type': 'ticker',
'sequence': 5928281348,
'product_id': 'BTC-USD',
'price': '8500.00000000',
'open_24h': '8217.24000000',
'volume_24h': '4529.13179472',
'low_24h': '8172.00000000',
'high_24h': '8600.00000000',
'volume_30d': '329178.93835825',
'best_bid': '8500',
'best_ask': '8500.01',
'side': 'sell',
'time': '2018-05-21T00:30:11.587000Z',
'trade_id': 43736677,
'last_size': '0.00241692'
}
'''
await self.callback(TICKER, Ticker(self.id, self.exchange_symbol_to_std_symbol(msg['product_id']), Decimal(msg['best_bid']), Decimal(msg['best_ask']), self.timestamp_normalize(msg['time']), raw=msg), timestamp)
async def _book_update(self, msg: dict, timestamp: float):
'''
{
'type': 'match', or last_match
'trade_id': 43736593
'maker_order_id': '2663b65f-b74e-4513-909d-975e3910cf22',
'taker_order_id': 'd058d737-87f1-4763-bbb4-c2ccf2a40bde',
'side': 'buy',
'size': '0.01235647',
'price': '8506.26000000',
'product_id': 'BTC-USD',
'sequence': 5928276661,
'time': '2018-05-21T00:26:05.585000Z'
}
'''
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
ts = self.timestamp_normalize(msg['time'])
if self.keep_l3_book and 'full' in self.subscription and pair in self.subscription['full']:
delta = {BID: [], ASK: []}
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
size = Decimal(msg['size'])
maker_order_id = msg['maker_order_id']
_, new_size = self.order_map[maker_order_id]
new_size -= size
if new_size <= 0:
del self.order_map[maker_order_id]
self.order_type_map.pop(maker_order_id, None)
delta[side].append((maker_order_id, price, 0))
del self._l3_book[pair].book[side][price][maker_order_id]
if len(self._l3_book[pair].book[side][price]) == 0:
del self._l3_book[pair].book[side][price]
else:
self.order_map[maker_order_id] = (price, new_size)
self._l3_book[pair].book[side][price][maker_order_id] = new_size
delta[side].append((maker_order_id, price, new_size))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=ts, delta=delta, raw=msg, sequence_number=self.seq_no[pair])
order_type = self.order_type_map.get(msg['taker_order_id'])
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(msg['product_id']),
SELL if msg['side'] == 'buy' else BUY,
Decimal(msg['size']),
Decimal(msg['price']),
ts,
id=str(msg['trade_id']),
type=order_type,
raw=msg
)
await self.callback(TRADES, t, timestamp)
async def _pair_level2_snapshot(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
bids = {Decimal(price): Decimal(amount) for price, amount in msg['bids']}
asks = {Decimal(price): Decimal(amount) for price, amount in msg['asks']}
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, bids=bids, asks=asks)
else:
self._l2_book[pair].book.bids = bids
self._l2_book[pair].book.asks = asks
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, raw=msg)
async def _pair_level2_update(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
ts = self.timestamp_normalize(msg['time'])
delta = {BID: [], ASK: []}
for side, price, amount in msg['changes']:
side = BID if side == 'buy' else ASK
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
del self._l2_book[pair].book[side][price]
delta[side].append((price, 0))
else:
self._l2_book[pair].book[side][price] = amount
delta[side].append((price, amount))
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=ts, raw=msg, delta=delta)
async def _book_snapshot(self, pairs: list):
# Coinbase needs some time to send messages to us
# before we request the snapshot. If we don't sleep
# the snapshot seq no could be much earlier than
# the subsequent messages, causing a seq no mismatch.
await asyncio.sleep(2)
urls = [self.rest_endpoints[0].route('l3book', self.sandbox).format(pair) for pair in pairs]
results = []
for url in urls:
ret = await self.http_conn.read(url)
results.append(ret)
# rate limit - 3 per second
await asyncio.sleep(0.3)
timestamp = time.time()
for res, pair in zip(results, pairs):
orders = json.loads(res, parse_float=Decimal)
npair = self.exchange_symbol_to_std_symbol(pair)
self._l3_book[npair] = OrderBook(self.id, pair, max_depth=self.max_depth)
self.seq_no[npair] = orders['sequence']
for side in (BID, ASK):
for price, size, order_id in orders[side + 's']:
price = Decimal(price)
size = Decimal(size)
if price in self._l3_book[npair].book[side]:
self._l3_book[npair].book[side][price][order_id] = size
else:
self._l3_book[npair].book[side][price] = {order_id: size}
self.order_map[order_id] = (price, size)
await self.book_callback(L3_BOOK, self._l3_book[npair], timestamp, raw=orders)
async def _open(self, msg: dict, timestamp: float):
if not self.keep_l3_book:
return
delta = {BID: [], ASK: []}
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
size = Decimal(msg['remaining_size'])
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
order_id = msg['order_id']
ts = self.timestamp_normalize(msg['time'])
if price in self._l3_book[pair].book[side]:
self._l3_book[pair].book[side][price][order_id] = size
else:
self._l3_book[pair].book[side][price] = {order_id: size}
self.order_map[order_id] = (price, size)
delta[side].append((order_id, price, size))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=ts, delta=delta, raw=msg, sequence_number=msg['sequence'])
async def _done(self, msg: dict, timestamp: float):
"""
per Coinbase API Docs:
A done message will be sent for received orders which are fully filled or canceled due
to self-trade prevention. There will be no open message for such orders. Done messages
for orders which are not on the book should be ignored when maintaining a real-time order book.
"""
if 'price' not in msg:
# market order life cycle: received -> done
self.order_type_map.pop(msg['order_id'], None)
self.order_map.pop(msg['order_id'], None)
return
order_id = msg['order_id']
self.order_type_map.pop(order_id, None)
if order_id not in self.order_map:
return
del self.order_map[order_id]
if self.keep_l3_book:
delta = {BID: [], ASK: []}
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
ts = self.timestamp_normalize(msg['time'])
del self._l3_book[pair].book[side][price][order_id]
if len(self._l3_book[pair].book[side][price]) == 0:
del self._l3_book[pair].book[side][price]
delta[side].append((order_id, price, 0))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, delta=delta, timestamp=ts, raw=msg, sequence_number=msg['sequence'])
async def _received(self, msg: dict, timestamp: float):
"""
per Coinbase docs:
A valid order has been received and is now active. This message is emitted for every single
valid order as soon as the matching engine receives it whether it fills immediately or not.
This message is the only time we receive the order type (limit vs market) for a given order,
so we keep it in a map by order ID.
"""
order_id = msg["order_id"]
order_type = msg["order_type"]
self.order_type_map[order_id] = order_type
async def _change(self, msg: dict, timestamp: float):
"""
Like done, these updates can be sent for orders that are not in the book. Per the docs:
Not all done or change messages will result in changing the order book. These messages will
be sent for received orders which are not yet on the order book. Do not alter
the order book for such messages, otherwise your order book will be incorrect.
"""
if not self.keep_l3_book:
return
delta = {BID: [], ASK: []}
if 'price' not in msg or not msg['price']:
return
order_id = msg['order_id']
if order_id not in self.order_map:
return
ts = self.timestamp_normalize(msg['time'])
price = Decimal(msg['price'])
side = ASK if msg['side'] == 'sell' else BID
new_size = Decimal(msg['new_size'])
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
self._l3_book[pair].book[side][price][order_id] = new_size
self.order_map[order_id] = (price, new_size)
delta[side].append((order_id, price, new_size))
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, delta=delta, timestamp=ts, raw=msg, sequence_number=msg['sequence'])
async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float):
# PERF perf_start(self.id, 'msg')
msg = json.loads(msg, parse_float=Decimal)
if self.seq_no:
if 'product_id' in msg and 'sequence' in msg:
pair = self.exchange_symbol_to_std_symbol(msg['product_id'])
if not self.seq_no.get(pair, None):
return
if msg['sequence'] <= self.seq_no[pair]:
return
if msg['sequence'] != self.seq_no[pair] + 1:
LOG.warning("%s: Missing sequence number detected for %s. Received %d, expected %d", self.id, pair, msg['sequence'], self.seq_no[pair] + 1)
LOG.warning("%s: Resetting data for %s", self.id, pair)
self.__reset(symbol=pair)
await self._book_snapshot([pair])
return
self.seq_no[pair] = msg['sequence']
if 'type' in msg:
if msg['type'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['type'] == 'match' or msg['type'] == 'last_match':
await self._book_update(msg, timestamp)
elif msg['type'] == 'snapshot':
await self._pair_level2_snapshot(msg, timestamp)
elif msg['type'] == 'l2update':
await self._pair_level2_update(msg, timestamp)
elif msg['type'] == 'open':
await self._open(msg, timestamp)
elif msg['type'] == 'done':
await self._done(msg, timestamp)
elif msg['type'] == 'change':
await self._change(msg, timestamp)
elif msg['type'] == 'received':
await self._received(msg, timestamp)
elif msg['type'] == 'activate':
pass
elif msg['type'] == 'subscriptions':
pass
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
# PERF perf_end(self.id, 'msg')
# PERF perf_log(self.id, 'msg')
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
await conn.write(json.dumps({"type": "subscribe",
"product_ids": self.subscription[chan],
"channels": [chan]
}))
chan = self.std_channel_to_exchange(L3_BOOK)
if chan in self.subscription:
await self._book_snapshot(self.subscription[chan])
class BinanceFutures(Binance, BinanceFuturesRestMixin):
id = BINANCE_FUTURES
websocket_endpoints = [
WebsocketEndpoint('wss://fstream.binance.com',
sandbox='wss://stream.binancefuture.com',
options={'compression': None})
]
rest_endpoints = [
RestEndpoint('https://fapi.binance.com',
sandbox='https://testnet.binancefuture.com',
routes=Routes(
'/fapi/v1/exchangeInfo',
l2book='/fapi/v1/depth?symbol={}&limit={}',
authentication='/fapi/v1/listenKey',
open_interest='/fapi/v1//openInterest?symbol={}'))
]
valid_depths = [5, 10, 20, 50, 100, 500, 1000]
valid_depth_intervals = {'100ms', '250ms', '500ms'}
websocket_channels = {
**Binance.websocket_channels, FUNDING: 'markPrice',
OPEN_INTEREST: 'open_interest',
LIQUIDATIONS: 'forceOrder',
POSITIONS: POSITIONS
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
base, info = super()._parse_symbol_data(data)
add = {}
for symbol, orig in base.items():
if "_" in orig:
continue
add[f"{symbol.replace('PERP', 'PINDEX')}"] = f"p{orig}"
base.update(add)
return base, info
def __init__(self, open_interest_interval=1.0, **kwargs):
"""
open_interest_interval: float
time in seconds between open_interest polls
"""
super().__init__(**kwargs)
self.open_interest_interval = open_interest_interval
def _connect_rest(self):
ret = []
for chan in set(self.subscription):
if chan == 'open_interest':
addrs = [
self.rest_endpoints[0].route(
'open_interest', sandbox=self.sandbox).format(pair)
for pair in self.subscription[chan]
]
ret.append((HTTPPoll(addrs,
self.id,
delay=60.0,
sleep=self.open_interest_interval,
proxy=self.http_proxy), self.subscribe,
self.message_handler, self.authenticate))
return ret
def _check_update_id(self, pair: str, msg: dict) -> bool:
if self._l2_book[pair].delta is None and msg[
'u'] < self.last_update_id[pair]:
return True
elif msg['U'] <= self.last_update_id[pair] <= msg['u']:
self.last_update_id[pair] = msg['u']
return False
elif self.last_update_id[pair] == msg['pu']:
self.last_update_id[pair] = msg['u']
return False
else:
self._reset()
LOG.warning("%s: Missing book update detected, resetting book",
self.id)
return True
async def _open_interest(self, msg: dict, timestamp: float):
"""
{
"openInterest": "10659.509",
"symbol": "BTCUSDT",
"time": 1589437530011 // Transaction time
}
"""
pair = msg['symbol']
oi = msg['openInterest']
if oi != self._open_interest_cache.get(pair, None):
o = OpenInterest(self.id,
self.exchange_symbol_to_std_symbol(pair),
Decimal(oi),
self.timestamp_normalize(msg['time']),
raw=msg)
await self.callback(OPEN_INTEREST, o, timestamp)
self._open_interest_cache[pair] = oi
async def _account_update(self, msg: dict, timestamp: float):
"""
{
"e": "ACCOUNT_UPDATE", // Event Type
"E": 1564745798939, // Event Time
"T": 1564745798938 , // Transaction
"a": // Update Data
{
"m":"ORDER", // Event reason type
"B":[ // Balances
{
"a":"USDT", // Asset
"wb":"122624.12345678", // Wallet Balance
"cw":"100.12345678", // Cross Wallet Balance
"bc":"50.12345678" // Balance Change except PnL and Commission
},
{
"a":"BUSD",
"wb":"1.00000000",
"cw":"0.00000000",
"bc":"-49.12345678"
}
],
"P":[
{
"s":"BTCUSDT", // Symbol
"pa":"0", // Position Amount
"ep":"0.00000", // Entry Price
"cr":"200", // (Pre-fee) Accumulated Realized
"up":"0", // Unrealized PnL
"mt":"isolated", // Margin Type
"iw":"0.00000000", // Isolated Wallet (if isolated position)
"ps":"BOTH" // Position Side
}，
{
"s":"BTCUSDT",
"pa":"20",
"ep":"6563.66500",
"cr":"0",
"up":"2850.21200",
"mt":"isolated",
"iw":"13200.70726908",
"ps":"LONG"
},
{
"s":"BTCUSDT",
"pa":"-10",
"ep":"6563.86000",
"cr":"-45.04000000",
"up":"-1423.15600",
"mt":"isolated",
"iw":"6570.42511771",
"ps":"SHORT"
}
]
}
}
"""
for balance in msg['a']['B']:
b = Balance(self.id,
balance['a'],
Decimal(balance['wb']),
None,
raw=msg)
await self.callback(BALANCES, b, timestamp)
for position in msg['a']['P']:
p = Position(self.id,
self.exchange_symbol_to_std_symbol(position['s']),
Decimal(position['pa']),
Decimal(position['ep']),
position['ps'].lower(),
Decimal(position['up']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(POSITIONS, p, timestamp)
async def _order_update(self, msg: dict, timestamp: float):
"""
{
"e":"ORDER_TRADE_UPDATE", // Event Type
"E":1568879465651, // Event Time
"T":1568879465650, // Transaction Time
"o":
{
"s":"BTCUSDT", // Symbol
"c":"TEST", // Client Order Id
// special client order id:
// starts with "autoclose-": liquidation order
// "adl_autoclose": ADL auto close order
"S":"SELL", // Side
"o":"TRAILING_STOP_MARKET", // Order Type
"f":"GTC", // Time in Force
"q":"0.001", // Original Quantity
"p":"0", // Original Price
"ap":"0", // Average Price
"sp":"7103.04", // Stop Price. Please ignore with TRAILING_STOP_MARKET order
"x":"NEW", // Execution Type
"X":"NEW", // Order Status
"i":8886774, // Order Id
"l":"0", // Order Last Filled Quantity
"z":"0", // Order Filled Accumulated Quantity
"L":"0", // Last Filled Price
"N":"USDT", // Commission Asset, will not push if no commission
"n":"0", // Commission, will not push if no commission
"T":1568879465651, // Order Trade Time
"t":0, // Trade Id
"b":"0", // Bids Notional
"a":"9.91", // Ask Notional
"m":false, // Is this trade the maker side?
"R":false, // Is this reduce only
"wt":"CONTRACT_PRICE", // Stop Price Working Type
"ot":"TRAILING_STOP_MARKET", // Original Order Type
"ps":"LONG", // Position Side
"cp":false, // If Close-All, pushed with conditional order
"AP":"7476.89", // Activation Price, only puhed with TRAILING_STOP_MARKET order
"cr":"5.0", // Callback Rate, only puhed with TRAILING_STOP_MARKET order
"rp":"0" // Realized Profit of the trade
}
}
"""
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(msg['o']['s']),
str(msg['o']['i']),
BUY if msg['o']['S'].lower() == 'buy' else SELL,
msg['o']['x'],
LIMIT if msg['o']['o'].lower() == 'limit' else
MARKET if msg['o']['o'].lower() == 'market' else None,
Decimal(msg['o']['ap']) if
not Decimal.is_zero(Decimal(msg['o']['ap'])) else None,
Decimal(msg['o']['q']),
Decimal(msg['o']['q']) - Decimal(msg['o']['z']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Handle REST endpoint messages first
if 'openInterest' in msg:
return await self._open_interest(msg, timestamp)
# Handle account updates from User Data Stream
if self.requires_authentication:
msg_type = msg.get('e')
if msg_type == 'ACCOUNT_UPDATE':
await self._account_update(msg, timestamp)
elif msg_type == 'ORDER_TRADE_UPDATE':
await self._order_update(msg, timestamp)
return
# Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>}
# streamName is of format <symbol>@<channel>
pair, _ = msg['stream'].split('@', 1)
msg = msg['data']
pair = pair.upper()
msg_type = msg.get('e')
if msg_type == 'bookTicker':
await self._ticker(msg, timestamp)
elif msg_type == 'depthUpdate':
await self._book(msg, pair, timestamp)
elif msg_type == 'aggTrade':
await self._trade(msg, timestamp)
elif msg_type == 'forceOrder':
await self._liquidations(msg, timestamp)
elif msg_type == 'markPriceUpdate':
await self._funding(msg, timestamp)
elif msg['e'] == 'kline':
await self._candle(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class BinanceDelivery(Binance, BinanceDeliveryRestMixin):
id = BINANCE_DELIVERY
# https://binance-docs.github.io/apidocs/delivery/en/#testnet
websocket_endpoints = [
WebsocketEndpoint('wss://dstream.binance.com',
options={'compression': None},
sandbox='wss://dstream.binancefuture.com')
]
rest_endpoints = [
RestEndpoint('https://dapi.binance.com',
routes=Routes('/dapi/v1/exchangeInfo',
l2book='/dapi/v1/depth?symbol={}&limit={}',
authentication='/dapi/v1/listenKey'),
sandbox='https://testnet.binancefuture.com')
]
valid_depths = [5, 10, 20, 50, 100, 500, 1000]
valid_depth_intervals = {'100ms', '250ms', '500ms'}
websocket_channels = {
**Binance.websocket_channels, FUNDING: 'markPrice',
OPEN_INTEREST: 'open_interest',
LIQUIDATIONS: 'forceOrder',
POSITIONS: POSITIONS
}
def _check_update_id(self, pair: str, msg: dict) -> bool:
if self._l2_book[pair].delta is None and msg[
'u'] < self.last_update_id[pair]:
return True
elif msg['U'] <= self.last_update_id[pair] <= msg['u']:
self.last_update_id[pair] = msg['u']
return False
elif self.last_update_id[pair] == msg['pu']:
self.last_update_id[pair] = msg['u']
return False
else:
self._reset()
LOG.warning("%s: Missing book update detected, resetting book",
self.id)
return True
async def _account_update(self, msg: dict, timestamp: float):
"""
{
"e": "ACCOUNT_UPDATE", // Event Type
"E": 1564745798939, // Event Time
"T": 1564745798938 , // Transaction
"i": "SfsR", // Account Alias
"a": // Update Data
{
"m":"ORDER", // Event reason type
"B":[ // Balances
{
"a":"BTC", // Asset
"wb":"122624.12345678", // Wallet Balance
"cw":"100.12345678" // Cross Wallet Balance
},
{
"a":"ETH",
"wb":"1.00000000",
"cw":"0.00000000"
}
],
"P":[
{
"s":"BTCUSD_200925", // Symbol
"pa":"0", // Position Amount
"ep":"0.0", // Entry Price
"cr":"200", // (Pre-fee) Accumulated Realized
"up":"0", // Unrealized PnL
"mt":"isolated", // Margin Type
"iw":"0.00000000", // Isolated Wallet (if isolated position)
"ps":"BOTH" // Position Side
},
{
"s":"BTCUSD_200925",
"pa":"20",
"ep":"6563.6",
"cr":"0",
"up":"2850.21200000",
"mt":"isolated",
"iw":"13200.70726908",
"ps":"LONG"
},
{
"s":"BTCUSD_200925",
"pa":"-10",
"ep":"6563.8",
"cr":"-45.04000000",
"up":"-1423.15600000",
"mt":"isolated",
"iw":"6570.42511771",
"ps":"SHORT"
}
]
}
}
"""
for balance in msg['a']['B']:
b = Balance(self.id,
balance['a'],
Decimal(balance['wb']),
None,
raw=msg)
await self.callback(BALANCES, b, timestamp)
for position in msg['a']['P']:
p = Position(self.id,
self.exchange_symbol_to_std_symbol(position['s']),
Decimal(position['pa']),
Decimal(position['ep']),
position['ps'].lower(),
Decimal(position['up']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(POSITIONS, p, timestamp)
async def _order_update(self, msg: dict, timestamp: float):
"""
{
"e":"ORDER_TRADE_UPDATE", // Event Type
"E":1591274595442, // Event Time
"T":1591274595453, // Transaction Time
"i":"SfsR", // Account Alias
"o":
{
"s":"BTCUSD_200925", // Symbol
"c":"TEST", // Client Order Id
// special client order id:
// starts with "autoclose-": liquidation order
// "adl_autoclose": ADL auto close order
"S":"SELL", // Side
"o":"TRAILING_STOP_MARKET", // Order Type
"f":"GTC", // Time in Force
"q":"2", // Original Quantity
"p":"0", // Original Price
"ap":"0", // Average Price
"sp":"9103.1", // Stop Price. Please ignore with TRAILING_STOP_MARKET order
"x":"NEW", // Execution Type
"X":"NEW", // Order Status
"i":8888888, // Order Id
"l":"0", // Order Last Filled Quantity
"z":"0", // Order Filled Accumulated Quantity
"L":"0", // Last Filled Price
"ma": "BTC", // Margin Asset
"N":"BTC", // Commission Asset of the trade, will not push if no commission
"n":"0", // Commission of the trade, will not push if no commission
"T":1591274595442, // Order Trade Time
"t":0, // Trade Id
"rp": "0", // Realized Profit of the trade
"b":"0", // Bid quantity of base asset
"a":"0", // Ask quantity of base asset
"m":false, // Is this trade the maker side?
"R":false, // Is this reduce only
"wt":"CONTRACT_PRICE", // Stop Price Working Type
"ot":"TRAILING_STOP_MARKET",// Original Order Type
"ps":"LONG", // Position Side
"cp":false, // If Close-All, pushed with conditional order
"AP":"9476.8", // Activation Price, only puhed with TRAILING_STOP_MARKET order
"cr":"5.0", // Callback Rate, only puhed with TRAILING_STOP_MARKET order
"pP": false // If conditional order trigger is protected
}
}
"""
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(msg['o']['s']),
str(msg['o']['i']),
BUY if msg['o']['S'].lower() == 'buy' else SELL,
msg['o']['x'],
LIMIT if msg['o']['o'].lower() == 'limit' else
MARKET if msg['o']['o'].lower() == 'market' else None,
Decimal(msg['o']['ap']) if
not Decimal.is_zero(Decimal(msg['o']['ap'])) else None,
Decimal(msg['o']['q']),
Decimal(msg['o']['q']) - Decimal(msg['o']['z']),
self.timestamp_normalize(msg['E']),
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Handle account updates from User Data Stream
if self.requires_authentication:
msg_type = msg.get('e')
if msg_type == 'ACCOUNT_UPDATE':
await self._account_update(msg, timestamp)
elif msg_type == 'ORDER_TRADE_UPDATE':
await self._order_update(msg, timestamp)
return
# Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>}
# streamName is of format <symbol>@<channel>
pair, _ = msg['stream'].split('@', 1)
msg = msg['data']
pair = pair.upper()
msg_type = msg.get('e')
if msg_type == 'bookTicker':
await self._ticker(msg, timestamp)
elif msg_type == 'depthUpdate':
await self._book(msg, pair, timestamp)
elif msg_type == 'aggTrade':
await self._trade(msg, timestamp)
elif msg_type == 'forceOrder':
await self._liquidations(msg, timestamp)
elif msg_type == 'markPriceUpdate':
await self._funding(msg, timestamp)
elif msg_type == 'kline':
await self._candle(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class FTX(Feed, FTXRestMixin):
id = FTX_id
allow_empty_subscriptions = True
websocket_endpoints = [
WebsocketEndpoint('wss://ftx.com/ws/', options={'compression': None})
]
rest_endpoints = [
RestEndpoint('https://ftx.com',
routes=Routes('/api/markets',
open_interest='/api/futures/{}/stats',
funding='/api/funding_rates?future={}',
stats='/api/futures/{}/stats'))
]
websocket_channels = {
L2_BOOK: 'orderbook',
TRADES: 'trades',
TICKER: 'ticker',
FUNDING: 'funding',
OPEN_INTEREST: 'open_interest',
LIQUIDATIONS: 'trades',
ORDER_INFO: 'orders',
FILLS: 'fills',
}
request_limit = 30
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for d in data['result']:
if not d['enabled']:
continue
expiry = None
stype = SPOT
# FTX Futures contracts are stable coin settled, but
# prices quoted are in USD, see https://help.ftx.com/hc/en-us/articles/360024780791-What-Are-Futures
if "-MOVE-" in d['name']:
stype = FUTURES
base, expiry = d['name'].rsplit("-", maxsplit=1)
quote = 'USD'
if 'Q' in expiry:
year, quarter = expiry.split("Q")
year = year[2:]
date = ["0325", "0624", "0924", "1231"]
expiry = year + date[int(quarter) - 1]
elif "-" in d['name']:
base, expiry = d['name'].split("-")
quote = 'USD'
stype = FUTURES
if expiry == 'PERP':
expiry = None
stype = PERPETUAL
elif d['type'] == SPOT:
base, quote = d['baseCurrency'], d['quoteCurrency']
else:
# not enough info to construct a symbol - this is usually caused
# by non crypto futures, i.e. TRUMP2024 or other contracts involving
# betting on world events
continue
s = Symbol(base, quote, type=stype, expiry_date=expiry)
ret[s.normalized] = d['name']
info['tick_size'][s.normalized] = d['priceIncrement']
info['quantity_step'][s.normalized] = d['sizeIncrement']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
self._funding_cache = {}
self._open_interest_cache = {}
async def generate_token(self, conn: AsyncConnection):
ts = int(time() * 1000)
msg = {
'op': 'login',
'args': {
'key':
self.key_id,
'sign':
hmac.new(self.key_secret.encode(),
f'{ts}websocket_login'.encode(),
'sha256').hexdigest(),
'time':
ts,
}
}
if self.subaccount:
msg['args']['subaccount'] = self.subaccount
await conn.write(json.dumps(msg))
async def authenticate(self, conn: AsyncConnection):
if self.requires_authentication:
await self.generate_token(conn)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
symbols = self.subscription[chan]
if chan == FUNDING:
asyncio.create_task(self._funding(symbols))
continue
if chan == OPEN_INTEREST:
asyncio.create_task(self._open_interest(symbols))
continue
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
await conn.write(
json.dumps({
"channel": chan,
"op": "subscribe"
}))
continue
for pair in symbols:
await conn.write(
json.dumps({
"channel": chan,
"market": pair,
"op": "subscribe"
}))
async def _open_interest(self, pairs: Iterable):
"""
{
"success": true,
"result": {
"volume": 1000.23,
"nextFundingRate": 0.00025,
"nextFundingTime": "2019-03-29T03:00:00+00:00",
"expirationPrice": 3992.1,
"predictedExpirationPrice": 3993.6,
"strikePrice": 8182.35,
"openInterest": 21124.583
}
}
"""
while True:
for pair in pairs:
# OI only for perp and futures, so check for / in pair name indicating spot
if '/' in pair:
continue
data = await self.http_conn.read(self.rest_endpoints[0].route(
'open_interest', sandbox=self.sandbox).format(pair))
received = time()
data = json.loads(data, parse_float=Decimal)
if 'result' in data:
oi = data['result']['openInterest']
if oi != self._open_interest_cache.get(pair, None):
o = OpenInterest(
self.id,
self.exchange_symbol_to_std_symbol(pair),
oi,
None,
raw=data)
await self.callback(OPEN_INTEREST, o, received)
self._open_interest_cache[pair] = oi
await asyncio.sleep(1)
await asyncio.sleep(60)
async def _funding(self, pairs: Iterable):
"""
{
"success": true,
"result": [
{
"future": "BTC-PERP",
"rate": 0.0025,
"time": "2019-06-02T08:00:00+00:00"
}
]
}
"""
while True:
for pair in pairs:
if '-PERP' not in pair:
continue
data = await self.http_conn.read(self.rest_endpoints[0].route(
'funding', sandbox=self.sandbox).format(pair))
data = json.loads(data, parse_float=Decimal)
data2 = await self.http_conn.read(self.rest_endpoints[0].route(
'stats', sandbox=self.sandbox).format(pair))
data2 = json.loads(data2, parse_float=Decimal)
received = time()
data['predicted_rate'] = Decimal(
data2['result']['nextFundingRate'])
last_update = self._funding_cache.get(pair, None)
update = str(data['result'][0]['rate']) + str(
data['result'][0]['time']) + str(data['predicted_rate'])
if last_update and last_update == update:
continue
else:
self._funding_cache[pair] = update
f = Funding(self.id,
self.exchange_symbol_to_std_symbol(
data['result'][0]['future']),
None,
data['result'][0]['rate'],
self.timestamp_normalize(
data2['result']['nextFundingTime']),
self.timestamp_normalize(
data['result'][0]['time']),
predicted_rate=data['predicted_rate'],
raw=[data, data2])
await self.callback(FUNDING, f, received)
await asyncio.sleep(0.1)
await asyncio.sleep(60)
async def _trade(self, msg: dict, timestamp: float):
"""
example message:
{"channel": "trades", "market": "BTC-PERP", "type": "update", "data": [{"id": null, "price": 10738.75,
"size": 0.3616, "side": "buy", "liquidation": false, "time": "2019-08-03T12:20:19.170586+00:00"}]}
"""
for trade in msg['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['market']),
BUY if trade['side'] == 'buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
float(self.timestamp_normalize(trade['time'])),
id=str(trade['id']),
raw=trade)
await self.callback(TRADES, t, timestamp)
if bool(trade['liquidation']):
liq = Liquidation(
self.id,
self.exchange_symbol_to_std_symbol(msg['market']),
BUY if trade['side'] == 'buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
str(trade['id']),
FILLED,
float(self.timestamp_normalize(trade['time'])),
raw=trade)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def _ticker(self, msg: dict, timestamp: float):
"""
example message:
{"channel": "ticker", "market": "BTC/USD", "type": "update", "data": {"bid": 10717.5, "ask": 10719.0,
"last": 10719.0, "time": 1564834587.1299787}}
"""
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(msg['market']),
Decimal(msg['data']['bid'] if msg['data']['bid'] else 0.0),
Decimal(msg['data']['ask'] if msg['data']['ask'] else 0.0),
float(msg['data']['time']),
raw=msg)
await self.callback(TICKER, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
"""
example messages:
snapshot:
{"channel": "orderbook", "market": "BTC/USD", "type": "partial", "data": {"time": 1564834586.3382702,
"checksum": 427503966, "bids": [[10717.5, 4.092], ...], "asks": [[10720.5, 15.3458], ...], "action": "partial"}}
update:
{"channel": "orderbook", "market": "BTC/USD", "type": "update", "data": {"time": 1564834587.1299787,
"checksum": 3115602423, "bids": [], "asks": [[10719.0, 14.7461]], "action": "update"}}
"""
check = msg['data']['checksum']
if msg['type'] == 'partial':
# snapshot
pair = self.exchange_symbol_to_std_symbol(msg['market'])
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth,
checksum_format='FTX')
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for price, amount in msg['data']['bids']
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for price, amount in msg['data']['asks']
}
if self.checksum_validation and self._l2_book[pair].book.checksum(
) != check:
raise BadChecksum
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=float(msg['data']['time']),
raw=msg,
checksum=check)
else:
# update
delta = {BID: [], ASK: []}
pair = self.exchange_symbol_to_std_symbol(msg['market'])
for side in ('bids', 'asks'):
s = BID if side == 'bids' else ASK
for price, amount in msg['data'][side]:
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
delta[s].append((price, 0))
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, amount))
self._l2_book[pair].book[s][price] = amount
if self.checksum_validation and self._l2_book[pair].book.checksum(
) != check:
raise BadChecksum
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=float(msg['data']['time']),
raw=msg,
checksum=check,
delta=delta)
async def _fill(self, msg: dict, timestamp: float):
"""
example message:
{
"channel": "fills",
"data": {
"fee": 78.05799225,
"feeRate": 0.0014,
"future": "BTC-PERP",
"id": 7828307,
"liquidity": "taker",
"market": "BTC-PERP",
"orderId": 38065410,
"tradeId": 19129310,
"price": 3723.75,
"side": "buy",
"size": 14.973,
"time": "2019-05-07T16:40:58.358438+00:00",
"type": "order"
},
"type": "update"
}
"""
fill = msg['data']
f = Fill(self.id,
self.exchange_symbol_to_std_symbol(fill['market']),
BUY if fill['side'] == 'buy' else SELL,
Decimal(fill['size']),
Decimal(fill['price']),
Decimal(fill['fee']),
str(fill['tradeId']),
str(fill['orderId']),
None,
TAKER if fill['liquidity'] == 'taker' else MAKER,
fill['time'].timestamp(),
account=self.subaccount,
raw=msg)
await self.callback(FILLS, f, timestamp)
async def _order(self, msg: dict, timestamp: float):
"""
example message:
{
"channel": "orders",
"data": {
"id": 24852229,
"clientId": null,
"market": "XRP-PERP",
"type": "limit",
"side": "buy",
"size": 42353.0,
"price": 0.2977,
"reduceOnly": false,
"ioc": false,
"postOnly": false,
"status": "closed",
"filledSize": 0.0,
"remainingSize": 0.0,
"avgFillPrice": 0.2978
},
"type": "update"
}
"""
order = msg['data']
status = order['status']
if status == 'new':
status = SUBMITTING
elif status == 'open':
status = OPEN
elif status == 'closed':
status = CLOSED
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(order['market']),
str(order['id']),
BUY if order['side'].lower() == 'buy' else SELL,
status,
LIMIT if order['type'] == 'limit' else MARKET,
Decimal(order['price']) if order['price'] else None,
Decimal(order['filledSize']),
Decimal(order['remainingSize']),
None,
account=self.subaccount,
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'type' in msg:
if msg['type'] == 'subscribed':
if 'market' in msg:
LOG.info('%s: Subscribed to %s channel for %s', self.id,
msg['channel'], msg['market'])
else:
LOG.info('%s: Subscribed to %s channel', self.id,
msg['channel'])
elif msg['type'] == 'error':
LOG.error('%s: Received error message %s', self.id, msg)
raise Exception('Error from %s: %s', self.id, msg)
elif 'channel' in msg:
if msg['channel'] == 'orderbook':
await self._book(msg, timestamp)
elif msg['channel'] == 'trades':
await self._trade(msg, timestamp)
elif msg['channel'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['channel'] == 'fills':
await self._fill(msg, timestamp)
elif msg['channel'] == 'orders':
await self._order(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
class OKCoin(Feed):
id = OKCOIN
websocket_endpoints = [WebsocketEndpoint('wss://real.okcoin.com:8443/ws/v3')]
rest_endpoints = [RestEndpoint('https://www.okcoin.com', routes=Routes('/api/spot/v3/instruments'))]
websocket_channels = {
L2_BOOK: 'spot/depth_l2_tbt',
TRADES: 'spot/trade',
TICKER: 'spot/ticker',
}
@classmethod
def timestamp_normalize(cls, ts) -> float:
return ts.timestamp()
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for e in data:
s = Symbol(e['base_currency'], e['quote_currency'])
ret[s.normalized] = e['instrument_id']
info['tick_size'][s.normalized] = e['tick_size']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
for symbol in self.subscription[chan]:
request = {"op": "subscribe", "args": f"{chan}:{symbol}"}
await conn.write(json.dumps(request))
async def _ticker(self, msg: dict, timestamp: float):
"""
{
'table': 'spot/ticker',
'data': [
{
'instrument_id': 'BTC-USD',
'last': '3977.74',
'best_bid': '3977.08',
'best_ask': '3978.73',
'open_24h': '3978.21',
'high_24h': '3995.43',
'low_24h': '3961.02',
'base_volume_24h': '248.245',
'quote_volume_24h': '988112.225861',
'timestamp': '2019-03-22T22:26:34.019Z'
}
]
}
"""
for update in msg['data']:
pair = update['instrument_id']
update_timestamp = self.timestamp_normalize(update['timestamp'])
t = Ticker(self.id, pair, Decimal(update['best_bid']) if update['best_bid'] else Decimal(0), Decimal(update['best_ask']) if update['best_ask'] else Decimal(0), update_timestamp, raw=update)
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{'table': 'spot/trade', 'data': [{'instrument_id': 'BTC-USD', 'price': '3977.44', 'side': 'buy', 'size': '0.0096', 'timestamp': '2019-03-22T22:45:44.578Z', 'trade_id': '486519521'}]}
"""
for trade in msg['data']:
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(trade['instrument_id']),
BUY if trade['side'] == 'buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
self.timestamp_normalize(trade['timestamp']),
id=trade['trade_id'],
raw=trade
)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
if msg['action'] == 'partial':
# snapshot
for update in msg['data']:
pair = self.exchange_symbol_to_std_symbol(update['instrument_id'])
bids = {Decimal(price): Decimal(amount) for price, amount, *_ in update['bids']}
asks = {Decimal(price): Decimal(amount) for price, amount, *_ in update['asks']}
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth, checksum_format='OKCOIN', bids=bids, asks=asks)
if self.checksum_validation and self._l2_book[pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(update['timestamp']), raw=msg, checksum=update['checksum'] & 0xFFFFFFFF)
else:
# update
for update in msg['data']:
delta = {BID: [], ASK: []}
pair = self.exchange_symbol_to_std_symbol(update['instrument_id'])
for side in ('bids', 'asks'):
s = BID if side == 'bids' else ASK
for price, amount, *_ in update[side]:
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
if price in self._l2_book[pair].book[s]:
delta[s].append((price, 0))
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, amount))
self._l2_book[pair].book[s][price] = amount
if self.checksum_validation and self._l2_book[pair].book.checksum() != (update['checksum'] & 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(update['timestamp']), raw=msg, delta=delta, checksum=update['checksum'] & 0xFFFFFFFF)
async def message_handler(self, msg: str, conn, timestamp: float):
# DEFLATE compression, no header
msg = zlib.decompress(msg, -15)
msg = json.loads(msg, parse_float=Decimal)
if 'event' in msg:
if msg['event'] == 'error':
LOG.error("%s: Error: %s", self.id, msg)
elif msg['event'] == 'subscribe':
pass
else:
LOG.warning("%s: Unhandled event %s", self.id, msg)
elif 'table' in msg:
if 'ticker' in msg['table']:
await self._ticker(msg, timestamp)
elif 'trade' in msg['table']:
await self._trade(msg, timestamp)
elif 'depth_l2_tbt' in msg['table']:
await self._book(msg, timestamp)
elif 'spot/order' in msg['table']:
await self._order(msg, timestamp)
else:
LOG.warning("%s: Unhandled message %s", self.id, msg)
else:
LOG.warning("%s: Unhandled message %s", self.id, msg)
class CryptoDotCom(Feed):
id = CRYPTODOTCOM
websocket_endpoints = [WebsocketEndpoint('wss://stream.crypto.com/v2/market')]
rest_endpoints = [RestEndpoint('https://api.crypto.com', routes=Routes('/v2/public/get-instruments'))]
websocket_channels = {
L2_BOOK: 'book',
TRADES: 'trade',
TICKER: 'ticker',
CANDLES: 'candlestick'
}
request_limit = 100
valid_candle_intervals = {'1m', '5m', '15m', '30m', '1h', '4h', '6h', '12h', '1d', '1w', '2w', '1M'}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data['result']['instruments']:
sym = Symbol(entry['base_currency'], entry['quote_currency'])
info['instrument_type'][sym.normalized] = sym.type
ret[sym.normalized] = entry['instrument_name']
return ret, info
def __reset(self):
self._l2_book = {}
async def _trades(self, msg: dict, timestamp: float):
'''
{
'instrument_name': 'BTC_USDT',
'subscription': 'trade.BTC_USDT',
'channel': 'trade',
'data': [
{
'dataTime': 1637630445449,
'd': 2006341810221954784,
's': 'BUY',
'p': Decimal('56504.25'),
'q': Decimal('0.003802'),
't': 1637630445448,
'i': 'BTC_USDT'
}
]
}
'''
for entry in msg['data']:
t = Trade(
self.id,
self.exchange_symbol_to_std_symbol(msg['instrument_name']),
BUY if entry['s'] == 'BUY' else SELL,
entry['q'],
entry['p'],
self.timestamp_normalize(entry['t']),
id=str(entry['d']),
raw=entry
)
await self.callback(TRADES, t, timestamp)
async def _ticker(self, msg: dict, timestamp: float):
'''
{
'instrument_name': 'BTC_USDT',
'subscription': 'ticker.BTC_USDT',
'channel': 'ticker',
'data': [
{
'i': 'BTC_USDT',
'b': Decimal('57689.90'),
'k': Decimal('57690.90'),
'a': Decimal('57690.90'),
't': 1637705099140,
'v': Decimal('46427.152283'),
'h': Decimal('57985.00'),
'l': Decimal('55313.95'),
'c': Decimal('1311.15')
}
]
}
'''
for entry in msg['data']:
await self.callback(TICKER, Ticker(self.id, self.exchange_symbol_to_std_symbol(entry['i']), entry['b'], entry['a'], self.timestamp_normalize(entry['t']), raw=entry), timestamp)
async def _candle(self, msg: dict, timestamp: float):
'''
{
'instrument_name': 'BTC_USDT',
'subscription': 'candlestick.14D.BTC_USDT',
'channel': 'candlestick',
'depth': 300,
'interval': '14D',
'data': [
{
't': 1636934400000,
'o': Decimal('65502.68'),
'h': Decimal('66336.25'),
'l': Decimal('55313.95'),
'c': Decimal('57582.1'),
'v': Decimal('366802.492134')
}
]
}
'''
interval = msg['interval']
if interval == '14D':
interval = '2w'
elif interval == '7D':
interval = '1w'
elif interval == '1D':
interval = '1d'
for entry in msg['data']:
c = Candle(self.id,
self.exchange_symbol_to_std_symbol(msg['instrument_name']),
entry['t'] / 1000,
entry['t'] / 1000 + timedelta_str_to_sec(interval) - 1,
interval,
None,
entry['o'],
entry['c'],
entry['h'],
entry['l'],
entry['v'],
None,
None,
raw=entry)
await self.callback(CANDLES, c, timestamp)
async def _book(self, msg: dict, timestamp: float):
'''
{
'instrument_name': 'BTC_USDT',
'subscription': 'book.BTC_USDT.150',
'channel': 'book',
'depth': 150,
'data': [
{
'bids': [
[Decimal('57553.03'), Decimal('0.481606'), 2],
[Decimal('57552.47'), Decimal('0.000418'), 1],
...
]
'asks': [
[Decimal('57555.44'), Decimal('0.343236'), 1],
[Decimal('57555.95'), Decimal('0.026062'), 1],
...
]
}
]
}
'''
pair = self.exchange_symbol_to_std_symbol(msg['instrument_name'])
for entry in msg['data']:
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id, pair, max_depth=self.max_depth)
self._l2_book[pair].book.bids = {price: amount for price, amount, _ in entry['bids']}
self._l2_book[pair].book.asks = {price: amount for price, amount, _ in entry['asks']}
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(entry['t']), raw=entry)
async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['method'] == 'public/heartbeat':
msg['method'] = 'public/respond-heartbeat'
await conn.write(json.dumps(msg))
return
if msg['code'] != 0:
LOG.warning("%s: Error received from exchange %s", self.id, msg)
return
channel = msg.get('result', {}).get('channel')
if channel == 'trade':
await self._trades(msg['result'], timestamp)
elif channel == 'ticker':
await self._ticker(msg['result'], timestamp)
elif channel == 'candlestick':
await self._candle(msg['result'], timestamp)
elif channel == 'book':
await self._book(msg['result'], timestamp)
elif channel is None:
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
# API Docs recommend a sleep between connect and subscription to avoid rate limiting
await asyncio.sleep(1)
for chan, symbols in self.subscription.items():
def sym(chan, symbol):
chan_s = self.exchange_channel_to_std(chan)
if chan_s == L2_BOOK:
return f"{chan}.{symbol}.150"
if chan_s == CANDLES:
interval = self.candle_interval
if self.candle_interval == '1d':
interval = '1D'
elif self.candle_interval == '1w':
interval = '7D'
elif self.candle_interval == '2w':
interval = '14D'
return f"{chan}.{interval}.{symbol}"
return f"{chan}.{symbol}"
await conn.write(json.dumps({"method": "subscribe",
"params": {
"channels": [sym(chan, symbol) for symbol in symbols]
}}))
await asyncio.sleep(1)
class Bitget(Feed):
id = BITGET
websocket_endpoints = [
WebsocketEndpoint('wss://ws.bitget.com/spot/v1/stream',
instrument_filter=('TYPE', (SPOT, ))),
WebsocketEndpoint('wss://ws.bitget.com/mix/v1/stream',
instrument_filter=('TYPE', (PERPETUAL, ))),
]
rest_endpoints = [
RestEndpoint('https://api.bitget.com',
instrument_filter=('TYPE', (SPOT, )),
routes=Routes('/api/spot/v1/public/products')),
RestEndpoint('https://api.bitget.com',
instrument_filter=('TYPE', (PERPETUAL, )),
routes=Routes([
'/api/mix/v1/market/contracts?productType=umcbl',
'/api/mix/v1/market/contracts?productType=dmcbl'
])),
]
valid_candle_intervals = {
'1m', '5m', '15m', '30m', '1h', '4h', '12h', '1d', '1w'
}
websocket_channels = {
L2_BOOK: 'books',
TRADES: 'trade',
TICKER: 'ticker',
CANDLES: 'candle',
ORDER_INFO: 'orders',
BALANCES: 'account',
POSITIONS: 'positions'
}
request_limit = 20
@classmethod
def timestamp_normalize(cls, ts: int) -> float:
return ts / 1000
@classmethod
def _parse_symbol_data(cls, data: Union[List, Dict]) -> Tuple[Dict, Dict]:
"""
contract types
umcbl USDT Unified Contract
dmcbl Quanto Swap Contract
sumcbl USDT Unified Contract Analog disk (naming makes no sense, but these are basically testnet coins)
sdmcbl Quanto Swap Contract Analog disk (naming makes no sense, but these are basically testnet coins)
"""
ret = {}
info = defaultdict(dict)
if isinstance(data, dict):
data = [data]
for d in data:
for entry in d['data']:
"""
Spot
{
"baseCoin":"ALPHA",
"makerFeeRate":"0.001",
"maxTradeAmount":"0",
"minTradeAmount":"2",
"priceScale":"4",
"quantityScale":"4",
"quoteCoin":"USDT",
"status":"online",
"symbol":"ALPHAUSDT_SPBL",
"symbolName":"ALPHAUSDT",
"takerFeeRate":"0.001"
}
"""
if "symbolName" in entry:
sym = Symbol(entry['baseCoin'], entry['quoteCoin'])
ret[sym.normalized] = entry['symbolName']
else:
sym = Symbol(entry['baseCoin'],
entry['quoteCoin'],
type=PERPETUAL)
ret[sym.normalized] = entry['symbol']
info['instrument_type'][sym.normalized] = sym.type
info['is_quanto'][
sym.normalized] = 'dmcbl' in entry['symbol'].lower()
return ret, info
def __reset(self, conn: AsyncConnection):
if self.std_channel_to_exchange(L2_BOOK) in conn.subscription:
for pair in conn.subscription[self.std_channel_to_exchange(
L2_BOOK)]:
std_pair = self.exchange_symbol_to_std_symbol(pair)
if std_pair in self._l2_book:
del self._l2_book[std_pair]
async def _ticker(self, msg: dict, timestamp: float, symbol: str):
"""
{
'action': 'snapshot',
'arg': {
'instType': 'sp',
'channel': 'ticker',
'instId': 'BTCUSDT'
},
'data': [
{
'instId': 'BTCUSDT',
'last': '46572.07',
'open24h': '46414.54',
'high24h': '46767.30',
'low24h': '46221.11',
'bestBid': '46556.590000',
'bestAsk': '46565.670000',
'baseVolume': '1927.0855',
'quoteVolume': '89120317.8812',
'ts': 1649013100029,
'labeId': 0
}
]
}
"""
key = 'ts'
if msg['arg']['instType'] == 'mc':
key = 'systemTime'
for entry in msg['data']:
# sometimes snapshots do not have bids/asks in them
if 'bestBid' not in entry or 'bestAsk' not in entry:
continue
t = Ticker(self.id,
symbol,
Decimal(entry['bestBid']),
Decimal(entry['bestAsk']),
self.timestamp_normalize(entry[key]),
raw=entry)
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float, symbol: str):
"""
{
'action': 'update',
'arg': {
'instType': 'sp',
'channel': 'trade',
'instId': 'BTCUSDT'
},
'data': [
['1649014224602', '46464.51', '0.0023', 'sell']
]
}
"""
for entry in msg['data']:
t = Trade(self.id,
symbol,
SELL if entry[3] == 'sell' else BUY,
Decimal(entry[2]),
Decimal(entry[1]),
self.timestamp_normalize(int(entry[0])),
raw=entry)
await self.callback(TRADES, t, timestamp)
async def _candle(self, msg: dict, timestamp: float, symbol: str):
'''
{
'action': 'update',
'arg': {
'instType': 'sp',
'channel': 'candle1m',
'instId': 'BTCUSDT'
},
'data': [['1649014920000', '46434.2', '46437.98', '46434.2', '46437.98', '0.9469']]
}
'''
for entry in msg['data']:
t = Candle(self.id,
symbol,
self.timestamp_normalize(int(entry[0])),
self.timestamp_normalize(int(entry[0])) +
timedelta_str_to_sec(self.candle_interval),
self.candle_interval,
None,
Decimal(entry[1]),
Decimal(entry[4]),
Decimal(entry[2]),
Decimal(entry[3]),
Decimal(entry[5]),
None,
self.timestamp_normalize(int(entry[0])),
raw=entry)
await self.callback(CANDLES, t, timestamp)
async def _book(self, msg: dict, timestamp: float, symbol: str):
data = msg['data'][0]
if msg['action'] == 'snapshot':
'''
{
'action': 'snapshot',
'arg': {
'instType': 'sp',
'channel': 'books',
'instId': 'BTCUSDT'
},
'data': [
{
'asks': [['46700.38', '0.0554'], ['46701.25', '0.0147'], ...
'bids': [['46686.68', '0.0032'], ['46684.75', '0.0161'], ...
'checksum': -393656186,
'ts': '1649021358917'
}
]
}
'''
bids = {
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}
asks = {
Decimal(price): Decimal(amount)
for price, amount in data['asks']
}
self._l2_book[symbol] = OrderBook(self.id,
symbol,
max_depth=self.max_depth,
bids=bids,
asks=asks,
checksum_format=self.id)
if self.checksum_validation and self._l2_book[
symbol].book.checksum() != (data['checksum'] & 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
timestamp,
checksum=data['checksum'],
timestamp=self.timestamp_normalize(
int(data['ts'])),
raw=msg)
else:
'''
{
'action': 'update',
'arg': {
'instType': 'sp',
'channel': 'books',
'instId': 'BTCUSDT'
},
'data': [
{
'asks': [['46701.25', '0'], ['46701.46', '0.0054'], ...
'bids': [['46687.67', '0.0531'], ['46686.22', '0'], ...
'checksum': -750266015,
'ts': '1649021359467'
}
]
}
'''
delta = {BID: [], ASK: []}
for side, key in ((BID, 'bids'), (ASK, 'asks')):
for price, size in data[key]:
price = Decimal(price)
size = Decimal(size)
delta[side].append((price, size))
if size == 0:
del self._l2_book[symbol].book[side][price]
else:
self._l2_book[symbol].book[side][price] = size
if self.checksum_validation and self._l2_book[
symbol].book.checksum() != (data['checksum'] & 0xFFFFFFFF):
raise BadChecksum
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
timestamp,
delta=delta,
checksum=data['checksum'],
timestamp=self.timestamp_normalize(
int(data['ts'])),
raw=msg)
async def _account(self, msg: dict, symbol: str, timestamp: float):
'''
spot
{
'action': 'snapshot',
'arg': {
'instType': 'spbl',
'channel': 'account',
'instId': 'BTCUSDT_SPBL'
},
'data': []
}
futures
{
'action': 'snapshot',
'arg': {
'instType': 'dmcbl',
'channel': 'account',
'instId': 'BTCUSD_DMCBL'
},
'data': [{
'marginCoin': 'BTC',
'locked': '0.00000000',
'available': '0.00000000',
'maxOpenPosAvailable': '0.00000000',
'maxTransferOut': '0.00000000',
'equity': '0.00000000',
'usdtEquity': '0.000000000000'
},
{
'marginCoin': 'ETH',
'locked': '0.00000000',
'available': '0.00000000',
'maxOpenPosAvailable': '0.00000000',
'maxTransferOut': '0.00000000',
'equity': '0.00000000',
'usdtEquity': '0.000000000000'
}]
}
'''
for entry in msg['data']:
b = Balance(self.id,
symbol,
Decimal(entry['available']),
Decimal(entry['locked']),
raw=entry)
await self.callback(BALANCES, b, timestamp)
async def _positions(self, msg: dict, symbol: str, timestamp: float):
'''
{
'action': 'snapshot',
'arg': {
'instType': 'sumcbl',
'channel': 'positions',
'instId': 'SBTCSUSDT_SUMCBL'
},
'data': [
{
'posId': '900434465966956544',
'instId': 'SBTCSUSDT_SUMCBL',
'instName': 'SBTCSUSDT',
'marginCoin': 'SUSDT',
'margin': '103.2987',
'marginMode': 'crossed',
'holdSide': 'long',
'holdMode': 'double_hold',
'total': '0.05',
'available': '0.05',
'locked': '0',
'averageOpenPrice': '41319.5',
'leverage': 20,
'achievedProfits': '0',
'upl': '0.518',
'uplRate': '0.005',
'liqPx': '0',
'keepMarginRate': '0.004',
'marginRate': '0.022209875738',
'cTime': '1650406226626',
'uTime': '1650406613064'
}
]
}
'''
# exchange, symbol, position, entry_price, side, unrealised_pnl, timestamp, raw=None):
for entry in msg['data']:
p = Position(self.id,
symbol,
Decimal(entry['total']),
Decimal(entry['averageOpenPrice']),
LONG if entry['holdSide'] == 'long' else SHORT,
Decimal(entry['upl']),
self.timestamp_normalize(int(entry['uTime'])),
raw=entry)
await self.callback(POSITIONS, p, timestamp)
def _status(self, status: str) -> str:
if status == 'new':
return OPEN
if status == 'partial-fill':
return PARTIAL
if status == 'full-fill':
return FILLED
if status == 'cancelled':
return CANCELLED
return status
async def _order(self, msg: dict, symbol: str, timestamp: float):
'''
{
'action': 'snapshot',
'arg': {
'instType': 'sumcbl',
'channel': 'orders',
'instId': 'default'
}, 'data': [
{
'accFillSz': '0',
'cTime': 1650407316266,
'clOrdId': '900439036248367104',
'force': 'normal',
'instId': 'SBTCSUSDT_SUMCBL',
'lever': '20',
'notionalUsd': '2065.175',
'ordId': '900439036185452544',
'ordType': 'market',
'orderFee': [
{'feeCcy': 'SUSDT', 'fee': '0'
}],
'posSide': 'long',
'px': '0',
'side': 'buy',
'status': 'new',
'sz': '0.05',
'tdMode': 'cross',
'tgtCcy': 'SUSDT',
'uTime': 1650407316266
}
]
}
filled:
{
'action': 'snapshot',
'arg': {
'instType': 'sumcbl',
'channel': 'orders',
'instId': 'default'
},
'data': [{
'accFillSz': '0.1',
'avgPx': '41400',
'cTime': 1650408010067,
'clOrdId': '900441946260676608',
'execType': 'T',
'fillFee': '-2.484',
'fillFeeCcy': 'SUSDT',
'fillNotionalUsd': '4140',
'fillPx': '41400',
'fillSz': '0.1',
'fillTime': '1650408010163',
'force': 'normal',
'instId': 'SBTCSUSDT_SUMCBL',
'lever': '20',
'notionalUsd': '4139.95',
'ordId': '900441946180984832',
'ordType': 'market',
'orderFee': [{'feeCcy': 'SUSDT', 'fee': '-2.484'}],
'pnl': '0',
'posSide': 'long',
'px': '0',
'side': 'buy',
'status': 'full-fill',
'sz': '0.1',
'tdMode': 'cross',
'tgtCcy': 'SUSDT',
'tradeId': '900441946663366657',
'uTime': 1650408010163
}]
}
'''
for entry in msg['data']:
o = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(entry['instId']),
entry['ordId'],
entry['side'],
self._status(entry['status']),
entry['ordType'],
Decimal(entry['px'] if 'fillPx' not in
entry else entry['fillPx']),
Decimal(entry['sz']),
Decimal(entry['sz']) - Decimal(entry['accFillSz']),
self.timestamp_normalize(int(entry['uTime'])),
client_order_id=entry['clOrdId'],
raw=entry)
await self.callback(ORDER_INFO, o, timestamp)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'event' in msg:
# {'event': 'subscribe', 'arg': {'instType': 'sp', 'channel': 'ticker', 'instId': 'BTCUSDT'}}
if msg['event'] == 'login' and msg['code'] == 0:
LOG.info("%s: Authenticated successfully", conn.uuid)
return
if msg['event'] == 'subscribe':
return
if msg['event'] == 'error':
LOG.error('%s: Error from exchange: %s', conn.uuid, msg)
return
symbol = msg['arg']['instId']
if symbol != 'default':
if msg['arg']['instType'] == 'mc':
if symbol.endswith('T'):
symbol = self.exchange_symbol_to_std_symbol(symbol +
"_UMCBL")
else:
symbol = self.exchange_symbol_to_std_symbol(symbol +
"_DMCBL")
elif msg['arg']['instType'] in {'dmcbl', 'umcbl'}:
symbol = self.exchange_symbol_to_std_symbol(symbol)
elif msg['arg']['instType'] == 'sp':
symbol = self.exchange_symbol_to_std_symbol(symbol)
else:
# SPBL
symbol = self.exchange_symbol_to_std_symbol(
symbol.split("_")[0])
if msg['arg']['channel'] == 'books':
await self._book(msg, timestamp, symbol)
elif msg['arg']['channel'] == 'ticker':
await self._ticker(msg, timestamp, symbol)
elif msg['arg']['channel'] == 'trade':
await self._trade(msg, timestamp, symbol)
elif msg['arg']['channel'].startswith('candle'):
await self._candle(msg, timestamp, symbol)
elif msg['arg']['channel'].startswith('account'):
await self._account(msg, symbol, timestamp)
elif msg['arg']['channel'].startswith('orders'):
await self._order(msg, symbol, timestamp)
elif msg['arg']['channel'].startswith('positions'):
await self._positions(msg, symbol, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _login(self, conn: AsyncConnection):
LOG.debug("%s: Attempting authentication", conn.uuid)
timestamp = int(time())
msg = f"{timestamp}GET/user/verify"
msg = hmac.new(bytes(self.key_secret, encoding='utf8'),
bytes(msg, encoding='utf-8'),
digestmod='sha256')
sign = str(base64.b64encode(msg.digest()), 'utf8')
await conn.write(
json.dumps({
"op":
"login",
"args": [{
"apiKey": self.key_id,
"passphrase": self.key_passphrase,
"timestamp": timestamp,
"sign": sign
}]
}))
async def subscribe(self, conn: AsyncConnection):
if self.key_id and self.key_passphrase and self.key_secret:
await self._login(conn)
self.__reset(conn)
args = []
interval = self.candle_interval
if interval[-1] != 'm':
interval[-1] = interval[-1].upper()
for chan, symbols in conn.subscription.items():
for s in symbols:
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(s))
if sym.type == SPOT:
if chan == 'positions': # positions not applicable on spot
continue
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
itype = 'spbl'
s += '_SPBL'
else:
itype = 'SP'
else:
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
itype = s.split('_')[-1]
if chan == 'orders':
s = 'default' # currently only supports 'default' for order channel on futures
else:
itype = 'MC'
s = s.split("_")[0]
d = {
'instType': itype,
'channel':
chan if chan != 'candle' else 'candle' + interval,
'instId': s
}
args.append(d)
await conn.write(json.dumps({"op": "subscribe", "args": args}))
class Gemini(Feed, GeminiRestMixin):
id = GEMINI
websocket_channels = {
L2_BOOK: L2_BOOK,
TRADES: TRADES,
ORDER_INFO: ORDER_INFO
}
websocket_endpoints = [
WebsocketEndpoint(
'wss://api.gemini.com/v2/marketdata/',
sandbox='wss://api.sandbox.gemini.com/v2/marketdata/',
channel_filter=[
websocket_channels[L2_BOOK], websocket_channels[TRADES]
]),
WebsocketEndpoint(
'wss://api.gemini.com/v1/order/events',
sandbox='wss://api.sandbox.gemini.com/v1/order/events',
channel_filter=[websocket_channels[ORDER_INFO]],
authentication=True)
]
rest_endpoints = [
RestEndpoint('https://api.gemini.com',
routes=Routes('/v1/symbols/details/{}',
currencies='/v1/symbols',
authentication='/v1/order/events'))
]
request_limit = 1
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _symbol_endpoint_prepare(cls,
ep: RestEndpoint) -> Union[List[str], str]:
ret = cls.http_sync.read(ep.route('currencies'),
json=True,
uuid=cls.id)
return [ep.route('instruments').format(currency) for currency in ret]
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for symbol in data:
if symbol['status'] == 'closed':
continue
s = Symbol(symbol['base_currency'], symbol['quote_currency'])
ret[s.normalized] = symbol['symbol']
info['tick_size'][s.normalized] = symbol['tick_size']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self, pairs):
for pair in pairs:
self._l2_book[self.exchange_symbol_to_std_symbol(
pair)] = OrderBook(self.id,
self.exchange_symbol_to_std_symbol(pair),
max_depth=self.max_depth)
def generate_token(self, payload=None) -> dict:
if not payload:
payload = {}
payload['request'] = self.rest_endpoints[0].routes.authentication
payload['nonce'] = int(time.time() * 1000)
if self.account_name:
payload['account'] = self.account_name
b64_payload = base64.b64encode(json.dumps(payload).encode('utf-8'))
signature = hmac.new(self.key_secret.encode('utf-8'), b64_payload,
hashlib.sha384).hexdigest()
return {
'X-GEMINI-PAYLOAD': b64_payload.decode(),
'X-GEMINI-APIKEY': self.key_id,
'X-GEMINI-SIGNATURE': signature
}
async def _book(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
# Gemini sends ALL data for the symbol, so if we don't actually want
# the book data, bail before parsing
if self.subscription and (
(L2_BOOK in self.subscription
and msg['symbol'] not in self.subscription[L2_BOOK])
or L2_BOOK not in self.subscription):
return
data = msg['changes']
forced = not len(self._l2_book[pair].book.bids)
delta = {BID: [], ASK: []}
for entry in data:
side = ASK if entry[0] == 'sell' else BID
price = Decimal(entry[1])
amount = Decimal(entry[2])
if amount == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
delta[side].append((price, 0))
else:
self._l2_book[pair].book[side][price] = amount
delta[side].append((price, amount))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
delta=delta if not forced else None,
raw=msg)
async def _trade(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
price = Decimal(msg['price'])
side = SELL if msg['side'] == 'sell' else BUY
amount = Decimal(msg['quantity'])
t = Trade(self.id,
pair,
side,
amount,
price,
self.timestamp_normalize(msg['timestamp']),
id=str(msg['event_id']),
raw=msg)
await self.callback(TRADES, t, timestamp)
async def _order(self, msg: dict, timestamp: float):
'''
[{
"type": "accepted",
"order_id": "109535951",
"event_id": "109535952",
"api_session": "UI",
"symbol": "btcusd",
"side": "buy",
"order_type": "exchange limit",
"timestamp": "1547742904",
"timestampms": 1547742904989,
"is_live": true,
"is_cancelled": false,
"is_hidden": false,
"original_amount": "1",
"price": "3592.00",
"socket_sequence": 13
}]
'''
if msg['type'] == "initial" or msg['type'] == "accepted":
status = SUBMITTING
elif msg['type'] == "fill":
status = FILLED
elif msg['type'] == 'booked':
status = OPEN
elif msg['type'] == 'rejected':
status = FAILED
elif msg['type'] == 'cancelled':
status = CANCELLED
else:
status = msg['type']
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(msg['symbol'].upper()),
msg['order_id'],
BUY if msg['side'].lower() == 'buy' else SELL,
status,
LIMIT if msg['order_type'] == 'exchange limit' else STOP_LIMIT,
Decimal(msg['price']),
Decimal(msg['executed_amount']),
Decimal(msg['remaining_amount']),
msg['timestampms'] / 1000.0,
raw=msg)
await self.callback(ORDER_INFO, oi, timestamp)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if isinstance(msg, list):
for entry in msg:
await self._order(entry, timestamp)
return
if 'type' not in msg:
LOG.warning('%s: Error from exchange %s', self.id, msg)
elif msg['type'] == 'l2_updates':
await self._book(msg, timestamp)
elif msg['type'] == 'trade':
await self._trade(msg, timestamp)
elif msg['type'] == 'heartbeat':
return
elif msg['type'] == 'subscription_ack':
LOG.info('%s: Authenticated successfully', self.id)
elif msg['type'] == 'auction_result' or msg[
'type'] == 'auction_indicative' or msg[
'type'] == 'auction_open':
return
else:
LOG.warning('%s: Invalid message type %s', self.id, msg)
async def _ws_authentication(self, address: str,
options: dict) -> Tuple[str, dict]:
header = self.generate_token()
symbols = []
for channel in self.subscription:
if self.is_authenticated_channel(channel):
symbols.extend(self.subscription.get(channel))
symbols = '&'.join([f"symbolFilter={s.lower()}" for s in symbols
]) # needs to match REST format (lower case)
options['extra_headers'] = header
return f'{address}?{symbols}', options
async def subscribe(self, conn: AsyncConnection):
if self.std_channel_to_exchange(ORDER_INFO) in conn.subscription:
return
symbols = list(set(itertools.chain(*conn.subscription.values())))
self.__reset(symbols)
await conn.write(
json.dumps({
"type": "subscribe",
"subscriptions": [{
"name": "l2",
"symbols": symbols
}]
}))
class Phemex(Feed):
id = PHEMEX
websocket_endpoints = [
WebsocketEndpoint('wss://phemex.com/ws',
sandbox='wss://testnet.phemex.com/ws',
limit=20)
]
rest_endpoints = [
RestEndpoint('https://api.phemex.com',
routes=Routes('/exchange/public/cfg/v2/products'))
]
price_scale = {}
valid_candle_intervals = ('1m', '5m', '15m', '30m', '1h', '4h', '1d', '1M',
'1Q', '1Y')
candle_interval_map = {
interval: second
for interval, second in zip(valid_candle_intervals, [
60, 300, 900, 1800, 3600, 14400, 86400, 604800, 2592000, 7776000,
31104000
])
}
websocket_channels = {
BALANCES: 'aop.subscribe',
L2_BOOK: 'orderbook.subscribe',
TRADES: 'trade.subscribe',
CANDLES: 'kline.subscribe',
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1_000_000_000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data['data']['products']:
if entry['status'] != 'Listed':
continue
stype = entry['type'].lower()
base, quote = entry['displaySymbol'].split(" / ")
s = Symbol(base, quote, type=stype)
ret[s.normalized] = entry['symbol']
info['tick_size'][s.normalized] = entry[
'tickSize'] if 'tickSize' in entry else entry['quoteTickSize']
info['instrument_type'][s.normalized] = stype
# the price scale for spot symbols is not reported via the API but it is documented
# here in the API docs: https://github.com/phemex/phemex-api-docs/blob/master/Public-Spot-API-en.md#spot-currency-and-symbols
# the default value for spot is 10^8
cls.price_scale[s.normalized] = 10**entry.get('priceScale', 8)
return ret, info
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Phemex only allows 5 connections, with 20 subscriptions per connection, check we arent over the limit
if sum(map(len, self.subscription.values())) > 100:
raise ValueError(
f"{self.id} only allows a maximum of 100 symbol/channel subscriptions"
)
def __reset(self, conn: AsyncConnection):
if self.std_channel_to_exchange(L2_BOOK) in conn.subscription:
for pair in conn.subscription[self.std_channel_to_exchange(
L2_BOOK)]:
std_pair = self.exchange_symbol_to_std_symbol(pair)
if std_pair in self._l2_book:
del self._l2_book[std_pair]
async def _book(self, msg: dict, timestamp: float):
"""
{
'book': {
'asks': [],
'bids': [
[345475000, 14340]
]
},
'depth': 30,
'sequence': 9047872983,
'symbol': 'BTCUSD',
'timestamp': 1625329629283990943,
'type': 'incremental'
}
"""
symbol = self.exchange_symbol_to_std_symbol(msg['symbol'])
ts = self.timestamp_normalize(msg['timestamp'])
delta = {BID: [], ASK: []}
if msg['type'] == 'snapshot':
delta = None
self._l2_book[symbol] = OrderBook(
self.id,
symbol,
max_depth=self.max_depth,
bids={
Decimal(entry[0]) / Decimal(self.price_scale[symbol]):
Decimal(entry[1])
for entry in msg['book']['bids']
},
asks={
Decimal(entry[0]) / Decimal(self.price_scale[symbol]):
Decimal(entry[1])
for entry in msg['book']['asks']
})
else:
for key, side in (('asks', ASK), ('bids', BID)):
for price, amount in msg['book'][key]:
price = Decimal(price) / Decimal(self.price_scale[symbol])
amount = Decimal(amount)
delta[side].append((price, amount))
if amount == 0:
# for some unknown reason deletes can be repeated in book updates
if price in self._l2_book[symbol].book[side]:
del self._l2_book[symbol].book[side][price]
else:
self._l2_book[symbol].book[side][price] = amount
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
timestamp,
timestamp=ts,
delta=delta)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'sequence': 9047166781,
'symbol': 'BTCUSD',
'trades': [
[1625326381255067545, 'Buy', 345890000, 323]
],
'type': 'incremental'
}
"""
symbol = self.exchange_symbol_to_std_symbol(msg['symbol'])
for ts, side, price, amount in msg['trades']:
t = Trade(self.id,
symbol,
BUY if side == 'Buy' else SELL,
Decimal(amount),
Decimal(price) / Decimal(self.price_scale[symbol]),
self.timestamp_normalize(ts),
raw=msg)
await self.callback(TRADES, t, timestamp)
async def _candle(self, msg: dict, timestamp: float):
"""
{
'kline': [
[1625332980, 60, 346285000, 346300000, 346390000, 346300000, 346390000, 49917, 144121225]
],
'sequence': 9048385626,
'symbol': 'BTCUSD',
'type': 'incremental'
}
"""
symbol = self.exchange_symbol_to_std_symbol(msg['symbol'])
for entry in msg['kline']:
ts, _, _, open, high, low, close, volume, _ = entry
c = Candle(self.id, symbol, ts,
ts + self.candle_interval_map[self.candle_interval],
self.candle_interval, None,
Decimal(open) / Decimal(self.price_scale[symbol]),
Decimal(close) / Decimal(self.price_scale[symbol]),
Decimal(high) / Decimal(self.price_scale[symbol]),
Decimal(low) / Decimal(self.price_scale[symbol]),
Decimal(volume), None, None)
await self.callback(CANDLES, c, timestamp)
async def _user_data(self, msg: dict, timestamp: float):
'''
snapshot:
{
"accounts":[
{
"accountBalanceEv":100000024,
"accountID":675340001,
"bonusBalanceEv":0,
"currency":"BTC",
"totalUsedBalanceEv":1222,
"userID":67534
}
],
"orders":[
{
"accountID":675340001,
"action":"New",
"actionBy":"ByUser",
"actionTimeNs":1573711481897337000,
"addedSeq":1110523,
"bonusChangedAmountEv":0,
"clOrdID":"uuid-1573711480091",
"closedPnlEv":0,
"closedSize":0,
"code":0,
"cumQty":2,
"cumValueEv":23018,
"curAccBalanceEv":100000005,
"curAssignedPosBalanceEv":0,
"curBonusBalanceEv":0,
"curLeverageEr":0,
"curPosSide":"Buy",
"curPosSize":2,
"curPosTerm":1,
"curPosValueEv":23018,
"curRiskLimitEv":10000000000,
"currency":"BTC",
"cxlRejReason":0,
"displayQty":2,
"execFeeEv":-5,
"execID":"92301512-7a79-5138-b582-ac185223727d",
"execPriceEp":86885000,
"execQty":2,
"execSeq":1131034,
"execStatus":"MakerFill",
"execValueEv":23018,
"feeRateEr":-25000,
"lastLiquidityInd":"AddedLiquidity",
"leavesQty":0,
"leavesValueEv":0,
"message":"No error",
"ordStatus":"Filled",
"ordType":"Limit",
"orderID":"e9a45803-0af8-41b7-9c63-9b7c417715d9",
"orderQty":2,
"pegOffsetValueEp":0,
"priceEp":86885000,
"relatedPosTerm":1,
"relatedReqNum":2,
"side":"Buy",
"stopLossEp":0,
"stopPxEp":0,
"symbol":"BTCUSD",
"takeProfitEp":0,
"timeInForce":"GoodTillCancel",
"tradeType":"Trade",
"transactTimeNs":1573712555309040417,
"userID":67534
},
{
"accountID":675340001,
"action":"New",
"actionBy":"ByUser",
"actionTimeNs":1573711490507067000,
"addedSeq":1110980,
"bonusChangedAmountEv":0,
"clOrdID":"uuid-1573711488668",
"closedPnlEv":0,
"closedSize":0,
"code":0,
"cumQty":3,
"cumValueEv":34530,
"curAccBalanceEv":100000013,
"curAssignedPosBalanceEv":0,
"curBonusBalanceEv":0,
"curLeverageEr":0,
"curPosSide":"Buy",
"curPosSize":5,
"curPosTerm":1,
"curPosValueEv":57548,
"curRiskLimitEv":10000000000,
"currency":"BTC",
"cxlRejReason":0,
"displayQty":3,
"execFeeEv":-8,
"execID":"80899855-5b95-55aa-b84e-8d1052f19886",
"execPriceEp":86880000,
"execQty":3,
"execSeq":1131408,
"execStatus":"MakerFill",
"execValueEv":34530,
"feeRateEr":-25000,
"lastLiquidityInd":"AddedLiquidity",
"leavesQty":0,
"leavesValueEv":0,
"message":"No error",
"ordStatus":"Filled",
"ordType":"Limit",
"orderID":"7e03cd6b-e45e-48d9-8937-8c6628e7a79d",
"orderQty":3,
"pegOffsetValueEp":0,
"priceEp":86880000,
"relatedPosTerm":1,
"relatedReqNum":3,
"side":"Buy",
"stopLossEp":0,
"stopPxEp":0,
"symbol":"BTCUSD",
"takeProfitEp":0,
"timeInForce":"GoodTillCancel",
"tradeType":"Trade",
"transactTimeNs":1573712559100655668,
"userID":67534
},
{
"accountID":675340001,
"action":"New",
"actionBy":"ByUser",
"actionTimeNs":1573711499282604000,
"addedSeq":1111025,
"bonusChangedAmountEv":0,
"clOrdID":"uuid-1573711497265",
"closedPnlEv":0,
"closedSize":0,
"code":0,
"cumQty":4,
"cumValueEv":46048,
"curAccBalanceEv":100000024,
"curAssignedPosBalanceEv":0,
"curBonusBalanceEv":0,
"curLeverageEr":0,
"curPosSide":"Buy",
"curPosSize":9,
"curPosTerm":1,
"curPosValueEv":103596,
"curRiskLimitEv":10000000000,
"currency":"BTC",
"cxlRejReason":0,
"displayQty":4,
"execFeeEv":-11,
"execID":"0be06645-90b8-5abe-8eb0-dca8e852f82f",
"execPriceEp":86865000,
"execQty":4,
"execSeq":1132422,
"execStatus":"MakerFill",
"execValueEv":46048,
"feeRateEr":-25000,
"lastLiquidityInd":"AddedLiquidity",
"leavesQty":0,
"leavesValueEv":0,
"message":"No error",
"ordStatus":"Filled",
"ordType":"Limit",
"orderID":"66753807-9204-443d-acf9-946d15d5bedb",
"orderQty":4,
"pegOffsetValueEp":0,
"priceEp":86865000,
"relatedPosTerm":1,
"relatedReqNum":4,
"side":"Buy",
"stopLossEp":0,
"stopPxEp":0,
"symbol":"BTCUSD",
"takeProfitEp":0,
"timeInForce":"GoodTillCancel",
"tradeType":"Trade",
"transactTimeNs":1573712618104628671,
"userID":67534
}
],
"positions":[
{
"accountID":675340001,
"assignedPosBalanceEv":0,
"avgEntryPriceEp":86875941,
"bankruptCommEv":75022,
"bankruptPriceEp":90000,
"buyLeavesQty":0,
"buyLeavesValueEv":0,
"buyValueToCostEr":1150750,
"createdAtNs":0,
"crossSharedBalanceEv":99998802,
"cumClosedPnlEv":0,
"cumFundingFeeEv":0,
"cumTransactFeeEv":-24,
"currency":"BTC",
"dataVer":4,
"deleveragePercentileEr":0,
"displayLeverageEr":1000000,
"estimatedOrdLossEv":0,
"execSeq":1132422,
"freeCostEv":0,
"freeQty":-9,
"initMarginReqEr":1000000,
"lastFundingTime":1573703858883133252,
"lastTermEndTime":0,
"leverageEr":0,
"liquidationPriceEp":90000,
"maintMarginReqEr":500000,
"makerFeeRateEr":0,
"markPriceEp":86786292,
"orderCostEv":0,
"posCostEv":1115,
"positionMarginEv":99925002,
"positionStatus":"Normal",
"riskLimitEv":10000000000,
"sellLeavesQty":0,
"sellLeavesValueEv":0,
"sellValueToCostEr":1149250,
"side":"Buy",
"size":9,
"symbol":"BTCUSD",
"takerFeeRateEr":0,
"term":1,
"transactTimeNs":1573712618104628671,
"unrealisedPnlEv":-107,
"updatedAtNs":0,
"usedBalanceEv":1222,
"userID":67534,
"valueEv":103596
}
],
"sequence":1310812,
"timestamp":1573716998131003833,
"type":"snapshot"
}
incremental update:
{
"accounts":[
{
"accountBalanceEv":99999989,
"accountID":675340001,
"bonusBalanceEv":0,
"currency":"BTC",
"totalUsedBalanceEv":1803,
"userID":67534
}
],
"orders":[
{
"accountID":675340001,
"action":"New",
"actionBy":"ByUser",
"actionTimeNs":1573717286765750000,
"addedSeq":1192303,
"bonusChangedAmountEv":0,
"clOrdID":"uuid-1573717284329",
"closedPnlEv":0,
"closedSize":0,
"code":0,
"cumQty":0,
"cumValueEv":0,
"curAccBalanceEv":100000024,
"curAssignedPosBalanceEv":0,
"curBonusBalanceEv":0,
"curLeverageEr":0,
"curPosSide":"Buy",
"curPosSize":9,
"curPosTerm":1,
"curPosValueEv":103596,
"curRiskLimitEv":10000000000,
"currency":"BTC",
"cxlRejReason":0,
"displayQty":4,
"execFeeEv":0,
"execID":"00000000-0000-0000-0000-000000000000",
"execPriceEp":0,
"execQty":0,
"execSeq":1192303,
"execStatus":"New",
"execValueEv":0,
"feeRateEr":0,
"leavesQty":4,
"leavesValueEv":46098,
"message":"No error",
"ordStatus":"New",
"ordType":"Limit",
"orderID":"e329ae87-ce80-439d-b0cf-ad65272ed44c",
"orderQty":4,
"pegOffsetValueEp":0,
"priceEp":86770000,
"relatedPosTerm":1,
"relatedReqNum":5,
"side":"Buy",
"stopLossEp":0,
"stopPxEp":0,
"symbol":"BTCUSD",
"takeProfitEp":0,
"timeInForce":"GoodTillCancel",
"transactTimeNs":1573717286765896560,
"userID":67534
},
{
"accountID":675340001,
"action":"New",
"actionBy":"ByUser",
"actionTimeNs":1573717286765750000,
"addedSeq":1192303,
"bonusChangedAmountEv":0,
"clOrdID":"uuid-1573717284329",
"closedPnlEv":0,
"closedSize":0,
"code":0,
"cumQty":4,
"cumValueEv":46098,
"curAccBalanceEv":99999989,
"curAssignedPosBalanceEv":0,
"curBonusBalanceEv":0,
"curLeverageEr":0,
"curPosSide":"Buy",
"curPosSize":13,
"curPosTerm":1,
"curPosValueEv":149694,
"curRiskLimitEv":10000000000,
"currency":"BTC",
"cxlRejReason":0,
"displayQty":4,
"execFeeEv":35,
"execID":"8d1848a2-5faf-52dd-be71-9fecbc8926be",
"execPriceEp":86770000,
"execQty":4,
"execSeq":1192303,
"execStatus":"TakerFill",
"execValueEv":46098,
"feeRateEr":75000,
"lastLiquidityInd":"RemovedLiquidity",
"leavesQty":0,
"leavesValueEv":0,
"message":"No error",
"ordStatus":"Filled",
"ordType":"Limit",
"orderID":"e329ae87-ce80-439d-b0cf-ad65272ed44c",
"orderQty":4,
"pegOffsetValueEp":0,
"priceEp":86770000,
"relatedPosTerm":1,
"relatedReqNum":5,
"side":"Buy",
"stopLossEp":0,
"stopPxEp":0,
"symbol":"BTCUSD",
"takeProfitEp":0,
"timeInForce":"GoodTillCancel",
"tradeType":"Trade",
"transactTimeNs":1573717286765896560,
"userID":67534
}
],
"positions":[
{
"accountID":675340001,
"assignedPosBalanceEv":0,
"avgEntryPriceEp":86843828,
"bankruptCommEv":75056,
"bankruptPriceEp":130000,
"buyLeavesQty":0,
"buyLeavesValueEv":0,
"buyValueToCostEr":1150750,
"createdAtNs":0,
"crossSharedBalanceEv":99998186,
"cumClosedPnlEv":0,
"cumFundingFeeEv":0,
"cumTransactFeeEv":11,
"currency":"BTC",
"dataVer":5,
"deleveragePercentileEr":0,
"displayLeverageEr":1000000,
"estimatedOrdLossEv":0,
"execSeq":1192303,
"freeCostEv":0,
"freeQty":-13,
"initMarginReqEr":1000000,
"lastFundingTime":1573703858883133252,
"lastTermEndTime":0,
"leverageEr":0,
"liquidationPriceEp":130000,
"maintMarginReqEr":500000,
"makerFeeRateEr":0,
"markPriceEp":86732335,
"orderCostEv":0,
"posCostEv":1611,
"positionMarginEv":99924933,
"positionStatus":"Normal",
"riskLimitEv":10000000000,
"sellLeavesQty":0,
"sellLeavesValueEv":0,
"sellValueToCostEr":1149250,
"side":"Buy",
"size":13,
"symbol":"BTCUSD",
"takerFeeRateEr":0,
"term":1,
"transactTimeNs":1573717286765896560,
"unrealisedPnlEv":-192,
"updatedAtNs":0,
"usedBalanceEv":1803,
"userID":67534,
"valueEv":149694
}
],
"sequence":1315725,
"timestamp":1573717286767188294,
"type":"incremental"
}
'''
for entry in msg['accounts']:
b = Balance(self.id,
entry['currency'],
Decimal(entry['accountBalanceEv']),
Decimal(entry['totalUsedBalanceEv']),
self.timestamp_normalize(msg['timestamp']),
raw=entry)
await self.callback(BALANCES, b, timestamp)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'id' in msg and msg['id'] == 100:
if not msg['error']:
LOG.info("%s: Auth request result: %s", conn.uuid,
msg['result']['status'])
msg = json.dumps({
"id":
101,
"method":
self.std_channel_to_exchange(BALANCES),
"params": []
})
LOG.debug(
f"{conn.uuid}: Subscribing to authenticated channels: {msg}"
)
await conn.write(msg)
else:
LOG.warning("%s: Auth unsuccessful: %s", conn.uuid, msg)
elif 'id' in msg and msg['id'] == 101:
if not msg['error']:
LOG.info("%s: Subscribe to auth channels request result: %s",
conn.uuid, msg['result']['status'])
else:
LOG.warning(f"{conn.uuid}: Subscription unsuccessful: {msg}")
elif 'id' in msg and msg['id'] == 1 and not msg['error']:
pass
elif 'accounts' in msg:
await self._user_data(msg, timestamp)
elif 'book' in msg:
await self._book(msg, timestamp)
elif 'trades' in msg:
await self._trade(msg, timestamp)
elif 'kline' in msg:
await self._candle(msg, timestamp)
elif 'result' in msg:
if 'error' in msg and msg['error'] is not None:
LOG.warning("%s: Error from exchange %s", conn.uuid, msg)
return
else:
LOG.warning("%s: Unhandled 'result' message: %s", conn.uuid,
msg)
else:
LOG.warning("%s: Invalid message type %s", conn.uuid, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset(conn)
for chan, symbols in conn.subscription.items():
if not self.exchange_channel_to_std(chan) == BALANCES:
for sym in symbols:
msg = {"id": 1, "method": chan, "params": [sym]}
if self.exchange_channel_to_std(chan) == CANDLES:
msg['params'] = [
*[sym],
self.candle_interval_map[self.candle_interval]
]
LOG.debug(
f"{conn.uuid}: Sending subscribe request to public channel: {msg}"
)
await conn.write(json.dumps(msg))
async def authenticate(self, conn: AsyncConnection):
if any(
self.is_authenticated_channel(
self.exchange_channel_to_std(chan))
for chan in self.subscription):
auth = json.dumps(self._auth(self.key_id, self.key_secret))
LOG.debug(
f"{conn.uuid}: Sending authentication request with message {auth}"
)
await conn.write(auth)
def _auth(self, key_id, key_secret, session_id=100):
# https://github.com/phemex/phemex-api-docs/blob/master/Public-Contract-API-en.md#api-user-authentication
expires = int((time.time() + 60))
signature = str(
hmac.new(bytes(key_secret, 'utf-8'),
bytes(f'{key_id}{expires}', 'utf-8'),
digestmod='sha256').hexdigest())
auth = {
"method": "user.auth",
"params": ["API", key_id, signature, expires],
"id": session_id
}
return auth
class Upbit(Feed, UpbitRestMixin):
id = UPBIT
websocket_endpoints = [
WebsocketEndpoint('wss://api.upbit.com/websocket/v1')
]
rest_endpoints = [
RestEndpoint('https://api.upbit.com', routes=Routes('/v1/market/all'))
]
websocket_channels = {
L2_BOOK: L2_BOOK,
TRADES: TRADES,
}
request_limit = 10
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
for entry in data:
quote, base = entry['market'].split("-")
s = Symbol(base, quote)
ret[s.normalized] = entry['market']
info['instrument_type'][s.normalized] = s.type
return ret, info
async def _trade(self, msg: dict, timestamp: float):
"""
Doc : https://docs.upbit.com/v1.0.7/reference#시세-체결-조회
{
'ty': 'trade' // Event type
'cd': 'KRW-BTC', // Symbol
'tp': 6759000.0, // Trade Price
'tv': 0.03243003, // Trade volume(amount)
'tms': 1584257228806, // Timestamp
'ttms': 1584257228000, // Trade Timestamp
'ab': 'BID', // 'BID' or 'ASK'
'cp': 64000.0, // Change of price
'pcp': 6823000.0, // Previous closing price
'sid': 1584257228000000, // Sequential ID
'st': 'SNAPSHOT', // 'SNAPSHOT' or 'REALTIME'
'td': '2020-03-15', // Trade date utc
'ttm': '07:27:08', // Trade time utc
'c': 'FALL', // Change - 'FALL' / 'RISE' / 'EVEN'
}
"""
price = Decimal(msg['tp'])
amount = Decimal(msg['tv'])
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['cd']),
BUY if msg['ab'] == 'BID' else SELL,
amount,
price,
self.timestamp_normalize(msg['ttms']),
id=str(msg['sid']),
raw=msg)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
"""
Doc : https://docs.upbit.com/v1.0.7/reference#시세-호가-정보orderbook-조회
Currently, Upbit orderbook api only provides 15 depth book state and does not support delta
{
'ty': 'orderbook' // Event type
'cd': 'KRW-BTC', // Symbol
'obu': [{'ap': 6727000.0, 'as': 0.4744314, 'bp': 6721000.0, 'bs': 0.0014551}, // orderbook units
{'ap': 6728000.0, 'as': 1.85862302, 'bp': 6719000.0, 'bs': 0.00926683},
{'ap': 6729000.0, 'as': 5.43556558, 'bp': 6718000.0, 'bs': 0.40908977},
{'ap': 6730000.0, 'as': 4.41993651, 'bp': 6717000.0, 'bs': 0.48052204},
{'ap': 6731000.0, 'as': 0.09207, 'bp': 6716000.0, 'bs': 6.52612927},
{'ap': 6732000.0, 'as': 1.42736812, 'bp': 6715000.0, 'bs': 610.45535023},
{'ap': 6734000.0, 'as': 0.173, 'bp': 6714000.0, 'bs': 1.09218395},
{'ap': 6735000.0, 'as': 1.08739294, 'bp': 6713000.0, 'bs': 0.46785444},
{'ap': 6737000.0, 'as': 3.34450006, 'bp': 6712000.0, 'bs': 0.01300915},
{'ap': 6738000.0, 'as': 0.26, 'bp': 6711000.0, 'bs': 0.24701799},
{'ap': 6739000.0, 'as': 0.086, 'bp': 6710000.0, 'bs': 1.97964014},
{'ap': 6740000.0, 'as': 0.00658782, 'bp': 6708000.0, 'bs': 0.0002},
{'ap': 6741000.0, 'as': 0.8004, 'bp': 6707000.0, 'bs': 0.02022364},
{'ap': 6742000.0, 'as': 0.11040396, 'bp': 6706000.0, 'bs': 0.29082183},
{'ap': 6743000.0, 'as': 1.1, 'bp': 6705000.0, 'bs': 0.94493254}],
'st': 'REALTIME', // Streaming type - 'REALTIME' or 'SNAPSHOT'
'tas': 20.67627941, // Total ask size for given 15 depth (not total ask order size)
'tbs': 622.93769692, // Total bid size for given 15 depth (not total bid order size)
'tms': 1584263923870, // Timestamp
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['cd'])
orderbook_timestamp = self.timestamp_normalize(msg['tms'])
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(unit['bp']): Decimal(unit['bs'])
for unit in msg['obu'] if unit['bp'] > 0
}
self._l2_book[pair].book.asks = {
Decimal(unit['ap']): Decimal(unit['as'])
for unit in msg['obu'] if unit['ap'] > 0
}
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=orderbook_timestamp,
raw=msg)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['ty'] == "trade":
await self._trade(msg, timestamp)
elif msg['ty'] == "orderbook":
await self._book(msg, timestamp)
else:
LOG.warning("%s: Unhandled message %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
"""
Doc : https://docs.upbit.com/docs/upbit-quotation-websocket
For subscription, ticket information is commonly required.
In order to reduce the data size, format parameter is set to 'SIMPLE' instead of 'DEFAULT'
Examples (Note that the positions of the base and quote currencies are swapped.)
1. In order to get TRADES of "BTC-KRW" and "XRP-BTC" markets.
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC","BTC-XRP"]}]
2. In order to get ORDERBOOK of "BTC-KRW" and "XRP-BTC" markets.
> [{"ticket":"UNIQUE_TICKET"},{"type":"orderbook","codes":["KRW-BTC","BTC-XRP"]}]
3. In order to get TRADES of "BTC-KRW" and ORDERBOOK of "ETH-KRW"
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]}]
4. In order to get TRADES of "BTC-KRW", ORDERBOOK of "ETH-KRW and TICKER of "EOS-KRW"
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]},{"type":"ticker", "codes":["KRW-EOS"]}]
5. In order to get TRADES of "BTC-KRW", ORDERBOOK of "ETH-KRW and TICKER of "EOS-KRW" with in shorter format
> [{"ticket":"UNIQUE_TICKET"},{"format":"SIMPLE"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]},{"type":"ticker", "codes":["KRW-EOS"]}]
"""
chans = [{"ticket": uuid.uuid4()}, {"format": "SIMPLE"}]
for chan in self.subscription:
codes = list(self.subscription[chan])
if chan == L2_BOOK:
chans.append({
"type": "orderbook",
"codes": codes,
'isOnlyRealtime': True
})
if chan == TRADES:
chans.append({
"type": "trade",
"codes": codes,
'isOnlyRealtime': True
})
await conn.write(json.dumps(chans))
class KuCoin(Feed):
id = KUCOIN
websocket_endpoints = None
rest_endpoints = [
RestEndpoint('https://api.kucoin.com',
routes=Routes(
'/api/v1/symbols',
l2book='/api/v3/market/orderbook/level2?symbol={}'))
]
valid_candle_intervals = {
'1m', '3m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d',
'1w'
}
candle_interval_map = {
'1m': '1min',
'3m': '3min',
'15m': '15min',
'30m': '30min',
'1h': '1hour',
'2h': '2hour',
'4h': '4hour',
'6h': '6hour',
'8h': '8hour',
'12h': '12hour',
'1d': '1day',
'1w': '1week'
}
websocket_channels = {
L2_BOOK: '/market/level2',
TRADES: '/market/match',
TICKER: '/market/ticker',
CANDLES: '/market/candles'
}
@classmethod
def is_authenticated_channel(cls, channel: str) -> bool:
return channel in (L2_BOOK)
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'tick_size': {}, 'instrument_type': {}}
for symbol in data['data']:
if not symbol['enableTrading']:
continue
s = Symbol(symbol['baseCurrency'], symbol['quoteCurrency'])
info['tick_size'][s.normalized] = symbol['priceIncrement']
ret[s.normalized] = symbol['symbol']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __init__(self, **kwargs):
address_info = self.http_sync.write(
'https://api.kucoin.com/api/v1/bullet-public', json=True)
token = address_info['data']['token']
address = address_info['data']['instanceServers'][0]['endpoint']
address = f"{address}?token={token}"
self.websocket_endpoints = [
WebsocketEndpoint(
address,
options={
'ping_interval':
address_info['data']['instanceServers'][0]['pingInterval']
/ 2000
})
]
super().__init__(**kwargs)
if any(
[len(self.subscription[chan]) > 300
for chan in self.subscription]):
raise ValueError(
"Kucoin has a limit of 300 symbols per connection")
self.__reset()
def __reset(self):
self._l2_book = {}
self.seq_no = {}
async def _candles(self, msg: dict, symbol: str, timestamp: float):
"""
{
'data': {
'symbol': 'BTC-USDT',
'candles': ['1619196960', '49885.4', '49821', '49890.5', '49821', '2.60137567', '129722.909001802'],
'time': 1619196997007846442
},
'subject': 'trade.candles.update',
'topic': '/market/candles:BTC-USDT_1min',
'type': 'message'
}
"""
symbol, interval = symbol.split("_")
interval = self.normalize_candle_interval[interval]
start, open, close, high, low, vol, _ = msg['data']['candles']
end = int(start) + timedelta_str_to_sec(interval) - 1
c = Candle(self.id,
symbol,
int(start),
end,
interval,
None,
Decimal(open),
Decimal(close),
Decimal(high),
Decimal(low),
Decimal(vol),
None,
msg['data']['time'] / 1000000000,
raw=msg)
await self.callback(CANDLES, c, timestamp)
async def _ticker(self, msg: dict, symbol: str, timestamp: float):
"""
{
"type":"message",
"topic":"/market/ticker:BTC-USDT",
"subject":"trade.ticker",
"data":{
"sequence":"1545896668986", // Sequence number
"price":"0.08", // Last traded price
"size":"0.011", // Last traded amount
"bestAsk":"0.08", // Best ask price
"bestAskSize":"0.18", // Best ask size
"bestBid":"0.049", // Best bid price
"bestBidSize":"0.036" // Best bid size
}
}
"""
t = Ticker(self.id,
symbol,
Decimal(msg['data']['bestBid']),
Decimal(msg['data']['bestAsk']),
None,
raw=msg)
await self.callback(TICKER, t, timestamp)
async def _trades(self, msg: dict, symbol: str, timestamp: float):
"""
{
"type":"message",
"topic":"/market/match:BTC-USDT",
"subject":"trade.l3match",
"data":{
"sequence":"1545896669145",
"type":"match",
"symbol":"BTC-USDT",
"side":"buy",
"price":"0.08200000000000000000",
"size":"0.01022222000000000000",
"tradeId":"5c24c5da03aa673885cd67aa",
"takerOrderId":"5c24c5d903aa6772d55b371e",
"makerOrderId":"5c2187d003aa677bd09d5c93",
"time":"1545913818099033203"
}
}
"""
t = Trade(self.id,
symbol,
BUY if msg['data']['side'] == 'buy' else SELL,
Decimal(msg['data']['size']),
Decimal(msg['data']['price']),
float(msg['data']['time']) / 1000000000,
id=msg['data']['tradeId'],
raw=msg)
await self.callback(TRADES, t, timestamp)
def generate_token(self, str_to_sign: str) -> dict:
# https://docs.kucoin.com/#authentication
# Now required to pass timestamp with string to sign. Timestamp should exactly match header timestamp
now = str(int(time.time() * 1000))
str_to_sign = now + str_to_sign
signature = base64.b64encode(
hmac.new(self.key_secret.encode('utf-8'),
str_to_sign.encode('utf-8'), hashlib.sha256).digest())
# Passphrase must now be encrypted by key_secret
passphrase = base64.b64encode(
hmac.new(self.key_secret.encode('utf-8'),
self.key_passphrase.encode('utf-8'),
hashlib.sha256).digest())
# API key version is currently 2 (whereas API version is anywhere from 1-3 ¯\_(ツ)_/¯)
header = {
"KC-API-KEY": self.key_id,
"KC-API-SIGN": signature.decode(),
"KC-API-TIMESTAMP": now,
"KC-API-PASSPHRASE": passphrase.decode(),
"KC-API-KEY-VERSION": "2"
}
return header
async def _snapshot(self, symbol: str):
str_to_sign = "GET" + self.rest_endpoints[0].routes.l2book.format(
symbol)
headers = self.generate_token(str_to_sign)
data = await self.http_conn.read(self.rest_endpoints[0].route(
'l2book', self.sandbox).format(symbol),
header=headers)
timestamp = time.time()
data = json.loads(data, parse_float=Decimal)
data = data['data']
self.seq_no[symbol] = int(data['sequence'])
bids = {
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}
asks = {
Decimal(price): Decimal(amount)
for price, amount in data['asks']
}
self._l2_book[symbol] = OrderBook(self.id,
symbol,
max_depth=self.max_depth,
bids=bids,
asks=asks)
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
timestamp,
raw=data,
sequence_number=int(data['sequence']))
async def _process_l2_book(self, msg: dict, symbol: str, timestamp: float):
"""
{
'data': {
'sequenceStart': 1615591136351,
'symbol': 'BTC-USDT',
'changes': {
'asks': [],
'bids': [['49746.9', '0.1488295', '1615591136351']]
},
'sequenceEnd': 1615591136351
},
'subject': 'trade.l2update',
'topic': '/market/level2:BTC-USDT',
'type': 'message'
}
"""
data = msg['data']
sequence = data['sequenceStart']
if symbol not in self._l2_book or sequence > self.seq_no[symbol] + 1:
if symbol in self.seq_no and sequence > self.seq_no[symbol] + 1:
LOG.warning("%s: Missing book update detected, resetting book",
self.id)
await self._snapshot(symbol)
data = msg['data']
if sequence < self.seq_no[symbol]:
return
self.seq_no[symbol] = data['sequenceEnd']
delta = {BID: [], ASK: []}
for s, side in (('bids', BID), ('asks', ASK)):
for update in data['changes'][s]:
price = Decimal(update[0])
amount = Decimal(update[1])
if amount == 0:
if price in self._l2_book[symbol].book[side]:
del self._l2_book[symbol].book[side][price]
delta[side].append((price, amount))
else:
self._l2_book[symbol].book[side][price] = amount
delta[side].append((price, amount))
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
timestamp,
delta=delta,
raw=msg,
sequence_number=data['sequenceEnd'])
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'topic' not in msg:
if msg['type'] == 'error':
LOG.warning("%s: error from exchange %s", self.id, msg)
return
elif msg['type'] in {'welcome', 'ack'}:
return
else:
LOG.warning("%s: Unhandled message type %s", self.id, msg)
return
topic, symbol = msg['topic'].split(":", 1)
topic = self.exchange_channel_to_std(topic)
if topic == TICKER:
await self._ticker(msg, symbol, timestamp)
elif topic == TRADES:
await self._trades(msg, symbol, timestamp)
elif topic == CANDLES:
await self._candles(msg, symbol, timestamp)
elif topic == L2_BOOK:
await self._process_l2_book(msg, symbol, timestamp)
else:
LOG.warning("%s: Unhandled message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
symbols = list(self.subscription[chan])
nchan = self.exchange_channel_to_std(chan)
if nchan == CANDLES:
for symbol in symbols:
await conn.write(
json.dumps({
'id': 1,
'type': 'subscribe',
'topic':
f"{chan}:{symbol}_{self.candle_interval_map[self.candle_interval]}",
'privateChannel': False,
'response': True
}))
else:
for slice_index in range(0, len(symbols), 100):
await conn.write(
json.dumps({
'id': 1,
'type': 'subscribe',
'topic':
f"{chan}:{','.join(symbols[slice_index: slice_index+100])}",
'privateChannel': False,
'response': True
}))
class HuobiSwap(HuobiDM):
id = HUOBI_SWAP
websocket_endpoints = [
WebsocketEndpoint('wss://api.hbdm.com/swap-ws',
instrument_filter=('QUOTE', ('USD', ))),
WebsocketEndpoint('wss://api.hbdm.com/linear-swap-ws',
instrument_filter=('QUOTE', ('USDT', )))
]
rest_endpoints = [
RestEndpoint(
'https://api.hbdm.com',
routes=Routes(
'/swap-api/v1/swap_contract_info',
funding='/swap-api/v1/swap_funding_rate?contract_code={}'),
instrument_filter=('QUOTE', ('USD', ))),
RestEndpoint(
'https://api.hbdm.com',
routes=Routes(
'/linear-swap-api/v1/swap_contract_info',
funding='/linear-swap-api/v1/swap_funding_rate?contract_code={}'
),
instrument_filter=('QUOTE', ('USDT', )))
]
websocket_channels = {**HuobiDM.websocket_channels, FUNDING: 'funding'}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for d in data:
for e in d['data']:
base, quote = e['contract_code'].split("-")
# Perpetual futures contract == perpetual swap
s = Symbol(base, quote, type=PERPETUAL)
ret[s.normalized] = e['contract_code']
info['tick_size'][s.normalized] = e['price_tick']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.funding_updates = {}
async def _funding(self, pairs):
"""
{
"status": "ok",
"data": {
"estimated_rate": "0.000100000000000000",
"funding_rate": "-0.000362360011416593",
"contract_code": "BTC-USD",
"symbol": "BTC",
"fee_asset": "BTC",
"funding_time": "1603872000000",
"next_funding_time": "1603900800000"
},
"ts": 1603866304635
}
"""
while True:
for pair in pairs:
# use symbol to look up correct endpoint
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(pair))
endpoint = None
for ep in self.rest_endpoints:
if sym.quote in ep.instrument_filter[1]:
endpoint = self.rest_endpoints[0].route(
'funding').format(pair)
data = await self.http_conn.read(endpoint)
data = json.loads(data, parse_float=Decimal)
received = time.time()
update = (data['data']['funding_rate'],
self.timestamp_normalize(
int(data['data']['next_funding_time'])))
if pair in self.funding_updates and self.funding_updates[
pair] == update:
await asyncio.sleep(1)
continue
self.funding_updates[pair] = update
f = Funding(self.id,
self.exchange_symbol_to_std_symbol(pair),
None,
Decimal(data['data']['funding_rate']),
self.timestamp_normalize(
int(data['data']['next_funding_time'])),
self.timestamp_normalize(
int(data['data']['funding_time'])),
predicted_rate=Decimal(
data['data']['estimated_rate']),
raw=data)
await self.callback(FUNDING, f, received)
await asyncio.sleep(0.1)
async def subscribe(self, conn: AsyncConnection):
if FUNDING in self.subscription:
loop = asyncio.get_event_loop()
loop.create_task(self._funding(self.subscription[FUNDING]))
await super().subscribe(conn)
class FMFW(Feed):
id = FMFW_id
websocket_endpoints = [
WebsocketEndpoint('wss://api.fmfw.io/api/3/ws/public')
]
rest_endpoints = [
RestEndpoint('https://api.fmfw.io',
routes=Routes('/api/3/public/symbol'))
]
valid_candle_intervals = {
'1m', '3m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '1M'
}
candle_interval_map = {
'1m': 'M1',
'3m': 'M3',
'5m': 'M5',
'15m': 'M15',
'30m': 'M30',
'1h': 'H1',
'4h': 'H4',
'1d': 'D1',
'1w': 'D7',
'1M': '1M'
}
websocket_channels = {
L2_BOOK: 'orderbook/full',
TRADES: 'trades',
TICKER: 'ticker/1s',
CANDLES: 'candles/'
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for sym, symbol in data.items():
s = Symbol(symbol['base_currency'], symbol['quote_currency'])
ret[s.normalized] = sym
info['tick_size'][s.normalized] = symbol['tick_size']
info['instrument_type'][s.normalized] = s.type
return ret, info
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
def __reset(self):
self._l2_book = {}
self.seq_no = {}
async def _book(self, msg: dict, ts: float):
if 'snapshot' in msg:
for pair, update in msg['snapshot'].items():
symbol = self.exchange_symbol_to_std_symbol(pair)
bids = {
Decimal(price): Decimal(size)
for price, size in update['b']
}
asks = {
Decimal(price): Decimal(size)
for price, size in update['a']
}
self._l2_book[symbol] = OrderBook(self.id,
pair,
max_depth=self.max_depth,
bids=bids,
asks=asks)
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
ts,
sequence_number=update['s'],
delta=None,
raw=msg,
timestamp=self.timestamp_normalize(
update['t']))
self.seq_no[symbol] = update['s']
else:
delta = {BID: [], ASK: []}
for pair, update in msg['update'].items():
symbol = self.exchange_symbol_to_std_symbol(pair)
if self.seq_no[symbol] + 1 != update['s']:
raise MissingSequenceNumber
self.seq_no[symbol] = update['s']
for side, key in ((BID, 'b'), (ASK, 'a')):
for price, size in update[key]:
price = Decimal(price)
size = Decimal(size)
delta[side].append((price, size))
if size == 0:
del self._l2_book[symbol].book[side][price]
else:
self._l2_book[symbol].book[side][price] = size
await self.book_callback(
L2_BOOK,
self._l2_book[symbol],
ts,
sequence_number=update['s'],
delta=delta,
raw=msg,
timestamp=self.timestamp_normalize(update['t']))
async def _trade(self, msg: dict, ts: float):
'''
{
'ch': 'trades',
'update': {
'BTCUSDT': [{
't': 1633803835228,
'i': 1633803835228,
'p': '54774.60',
'q': '0.00004',
's': 'buy'
}]
}
}
'''
for pair, update in msg['update'].items():
symbol = self.exchange_symbol_to_std_symbol(pair)
for trade in update:
t = Trade(self.id,
symbol,
BUY if trade['s'] == 'buy' else SELL,
Decimal(trade['q']),
Decimal(trade['p']),
self.timestamp_normalize(trade['t']),
id=str(trade['i']),
raw=msg)
await self.callback(TRADES, t, ts)
async def _ticker(self, msg: dict, ts: float):
'''
{
'ch': 'ticker/1s',
'data': {
'BTCUSDT': {
't': 1633804289795,
'a': '54813.56',
'A': '0.82000',
'b': '54810.31',
'B': '0.00660',
'o': '54517.48',
'c': '54829.88',
'h': '55493.92',
'l': '53685.61',
'v': '19025.22558',
'q': '1040244549.4048389',
'p': '312.40',
'P': '0.5730272198935094',
'L': 1417964345
}
}
}
'''
for sym, ticker in msg['data'].items():
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(sym),
Decimal(ticker['b']),
Decimal(ticker['a']),
self.timestamp_normalize(ticker['t']),
raw=msg)
await self.callback(TICKER, t, ts)
async def _candle(self, msg: dict, ts: float):
'''
{
'ch': 'candles/M1',
'update': {
'BTCUSDT': [{
't': 1633805940000,
'o': '54849.03',
'c': '54849.03',
'h': '54849.03',
'l': '54849.03',
'v': '0.00766',
'q': '420.1435698'
}]
}
}
'''
interval = msg['ch'].split("/")[-1]
for sym, updates in msg['update'].items():
symbol = self.exchange_symbol_to_std_symbol(sym)
for u in updates:
c = Candle(self.id,
symbol,
u['t'] / 1000,
u['t'] / 1000 + timedelta_str_to_sec(
self.normalize_candle_interval[interval]) - 0.1,
self.normalize_candle_interval[interval],
None,
Decimal(u['o']),
Decimal(u['c']),
Decimal(u['h']),
Decimal(u['l']),
Decimal(u['v']),
None,
self.timestamp_normalize(u['t']),
raw=msg)
await self.callback(CANDLES, c, ts)
async def message_handler(self, msg: str, conn, ts: float):
msg = json.loads(msg, parse_float=Decimal)
if 'result' in msg:
LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg)
elif msg['ch'] == 'orderbook/full':
await self._book(msg, ts)
elif msg['ch'] == 'trades':
await self._trade(msg, ts)
elif msg['ch'] == 'ticker/1s':
await self._ticker(msg, ts)
elif msg['ch'].startswith('candles/'):
await self._candle(msg, ts)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn):
self.__reset()
for chan in self.subscription:
await conn.write(
json.dumps({
"method":
"subscribe",
"params": {
"symbols": self.subscription[chan]
},
"ch":
chan if chan != 'candles/' else chan +
self.candle_interval_map[self.candle_interval],
"id":
1234
}))
class Bitfinex(Feed, BitfinexRestMixin):
id = BITFINEX
websocket_endpoints = [
WebsocketEndpoint('wss://api.bitfinex.com/ws/2', limit=25)
]
rest_endpoints = [
RestEndpoint('https://api-pub.bitfinex.com',
routes=Routes([
'/v2/conf/pub:list:pair:exchange',
'/v2/conf/pub:list:currency',
'/v2/conf/pub:list:pair:futures'
]))
]
websocket_channels = {
L3_BOOK: 'book-R0-{}-{}',
L2_BOOK: 'book-P0-{}-{}',
TRADES: 'trades',
TICKER: 'ticker',
}
request_limit = 1
valid_candle_intervals = {
'1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1d', '1w', '2w',
'1M'
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
# https://docs.bitfinex.com/docs/ws-general#supported-pairs
ret = {}
info = {'instrument_type': {}}
pairs = data[0][0]
currencies = data[1][0]
perpetuals = data[2][0]
for c in currencies:
c = c.replace('BCHN',
'BCH') # Bitfinex uses BCHN, other exchanges use BCH
c = c.replace('UST', 'USDT')
s = Symbol(c, c, type=CURRENCY)
ret[s.normalized] = "f" + c
info['instrument_type'][s.normalized] = CURRENCY
for p in pairs:
norm = p.replace('BCHN', 'BCH')
norm = norm.replace('UST', 'USDT')
if ':' in norm:
base, quote = norm.split(":")
else:
base, quote = norm[:3], norm[3:]
s = Symbol(base, quote)
ret[s.normalized] = "t" + p
info['instrument_type'][s.normalized] = s.type
for f in perpetuals:
norm = f.replace('BCHN', 'BCH')
norm = norm.replace('UST', 'USDT')
base, quote = norm.split(':') # 'ALGF0:USTF0'
base, quote = base[:-2], quote[:-2]
s = Symbol(base, quote, type=PERPETUAL)
ret[s.normalized] = "t" + f
info['instrument_type'][s.normalized] = s.type
return ret, info
def __init__(self,
symbols=None,
channels=None,
subscription=None,
number_of_price_points: int = 100,
book_frequency: str = 'F0',
**kwargs):
if number_of_price_points not in {1, 25, 100, 250}:
raise ValueError(
"number_of_price_points should be one of 1, 25, 100, 250")
if book_frequency not in {'F0', 'F1'}:
raise ValueError("book_frequency should be one of F0, F1")
super().__init__(symbols=symbols,
channels=channels,
subscription=subscription,
**kwargs)
self.number_of_price_points = number_of_price_points
self.book_frequency = book_frequency
if channels or subscription:
for chan in set(channels or subscription):
for pair in set(
subscription[chan] if subscription else symbols or []):
exch_sym = self.std_symbol_to_exchange_symbol(pair)
if (exch_sym[0] == 'f') == (chan != FUNDING):
LOG.warning(
'%s: No %s for symbol %s => Cryptofeed will subscribe to the wrong channel',
self.id, chan, pair)
self.handlers = {} # maps a channel id (int) to a function
self.order_map = defaultdict(dict)
self.seq_no = defaultdict(int)
def __reset(self, conn: AsyncConnection):
if self.std_channel_to_exchange(L2_BOOK) in conn.subscription:
for pair in conn.subscription[self.std_channel_to_exchange(
L2_BOOK)]:
std_pair = self.exchange_symbol_to_std_symbol(pair)
if std_pair in self._l2_book:
del self._l2_book[std_pair]
if conn.uuid in self.seq_no:
del self.seq_no[conn.uuid]
if self.std_channel_to_exchange(L3_BOOK) in conn.subscription:
for pair in conn.subscription[self.std_channel_to_exchange(
L3_BOOK)]:
std_pair = self.exchange_symbol_to_std_symbol(pair)
if std_pair in self._l3_book:
del self._l3_book[std_pair]
if std_pair in self.order_map:
del self.order_map[std_pair]
async def _ticker(self, pair: str, msg: list, timestamp: float):
if msg[1] == 'hb':
return # ignore heartbeats
# bid, bid_size, ask, ask_size, daily_change, daily_change_percent,
# last_price, volume, high, low
bid, _, ask, _, _, _, _, _, _, _ = msg[1]
t = Ticker(self.id, pair, Decimal(bid), Decimal(ask), None, raw=msg)
await self.callback(TICKER, t, timestamp)
async def _funding(self, pair: str, msg: list, timestamp: float):
async def _funding_update(funding: list, timestamp: float):
order_id, ts, amount, price, period = funding
t = Trade(self.id,
pair,
SELL if amount < 0 else BUY,
Decimal(abs(Decimal(amount))),
Decimal(price),
self.timestamp_normalize(ts),
id=order_id,
raw=funding)
await self.callback(TRADES, t, timestamp)
if isinstance(msg[1], list):
# snapshot
for funding in msg[1]:
await _funding_update(funding, timestamp)
elif msg[1] in ('te', 'fte'):
# update
await _funding_update(msg[2], timestamp)
elif msg[1] not in ('tu', 'ftu', 'hb'):
# ignore trade updates and heartbeats
LOG.warning('%s %s: Unexpected funding message %s', self.id, pair,
msg)
async def _trades(self, pair: str, msg: list, timestamp: float):
async def _trade_update(trade: list, timestamp: float):
order_id, ts, amount, price = trade
t = Trade(
self.id,
pair,
SELL if amount < 0 else BUY,
Decimal(abs(Decimal(amount))),
Decimal(price),
self.timestamp_normalize(ts),
id=str(order_id),
)
await self.callback(TRADES, t, timestamp)
if isinstance(msg[1], list):
# snapshot
for trade in msg[1]:
await _trade_update(trade, timestamp)
elif msg[1] in ('te', 'fte'):
# update
await _trade_update(msg[2], timestamp)
elif msg[1] not in ('tu', 'ftu', 'hb'):
# ignore trade updates and heartbeats
LOG.warning('%s %s: Unexpected trade message %s', self.id, pair,
msg)
async def _book(self, pair: str, msg: list, timestamp: float):
"""For L2 book updates."""
if not isinstance(msg[1], list):
if msg[1] != 'hb':
LOG.warning('%s: Unexpected book L2 msg %s', self.id, msg)
return
delta = None
if isinstance(msg[1][0], list):
# snapshot so clear book
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
for update in msg[1]:
price, _, amount = update
price = Decimal(price)
amount = Decimal(amount)
if amount > 0:
side = BID
else:
side = ASK
amount = abs(amount)
self._l2_book[pair].book[side][price] = amount
else:
# book update
delta = {BID: [], ASK: []}
price, count, amount = msg[1]
price = Decimal(price)
amount = Decimal(amount)
if amount > 0:
side = BID
else:
side = ASK
amount = abs(amount)
if count > 0:
# change at price level
delta[side].append((price, amount))
self._l2_book[pair].book[side][price] = amount
else:
# remove price level
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
delta[side].append((price, 0))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
delta=delta,
sequence_number=msg[-1])
async def _raw_book(self, pair: str, msg: list, timestamp: float):
"""For L3 book updates."""
if not isinstance(msg[1], list):
if msg[1] != 'hb':
LOG.warning('%s: Unexpected book L3 msg %s', self.id, msg)
return
def add_to_book(side, price, order_id, amount):
if price in self._l3_book[pair].book[side]:
self._l3_book[pair].book[side][price][order_id] = amount
else:
self._l3_book[pair].book[side][price] = {order_id: amount}
def remove_from_book(side, order_id):
price = self.order_map[pair][side][order_id]['price']
del self._l3_book[pair].book[side][price][order_id]
if len(self._l3_book[pair].book[side][price]) == 0:
del self._l3_book[pair].book[side][price]
delta = {BID: [], ASK: []}
if isinstance(msg[1][0], list):
# snapshot so clear orders
self.order_map[pair][BID] = {}
self.order_map[pair][ASK] = {}
self._l3_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
for update in msg[1]:
order_id, price, amount = update
price = Decimal(price)
amount = Decimal(amount)
if amount > 0:
side = BID
else:
side = ASK
amount = -amount
self.order_map[pair][side][order_id] = {
'price': price,
'amount': amount
}
add_to_book(side, price, order_id, amount)
else:
# book update
order_id, price, amount = msg[1]
price = Decimal(price)
amount = Decimal(amount)
if amount > 0:
side = BID
else:
side = ASK
amount = abs(amount)
if price == 0:
price = self.order_map[pair][side][order_id]['price']
remove_from_book(side, order_id)
del self.order_map[pair][side][order_id]
delta[side].append((order_id, price, 0))
else:
if order_id in self.order_map[pair][side]:
del_price = self.order_map[pair][side][order_id]['price']
delta[side].append((order_id, del_price, 0))
# remove existing order before adding new one
delta[side].append((order_id, price, amount))
remove_from_book(side, order_id)
else:
delta[side].append((order_id, price, amount))
add_to_book(side, price, order_id, amount)
self.order_map[pair][side][order_id] = {
'price': price,
'amount': amount
}
await self.book_callback(L3_BOOK,
self._l3_book[pair],
timestamp,
raw=msg,
delta=delta,
sequence_number=msg[-1])
@staticmethod
async def _do_nothing(msg: list, timestamp: float):
pass
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if isinstance(msg, list):
hb_skip = False
chan_handler = self.handlers.get(msg[0])
if chan_handler is None:
if msg[1] == 'hb':
hb_skip = True
else:
LOG.warning('%s: Unregistered channel ID in message %s',
conn.uuid, msg)
return
seq_no = msg[-1]
expected = self.seq_no[conn.uuid] + 1
if seq_no != expected:
LOG.warning(
'%s: missed message (sequence number) received %d, expected %d',
conn.uuid, seq_no, expected)
raise MissingSequenceNumber
self.seq_no[conn.uuid] = seq_no
if hb_skip:
return
await chan_handler(msg, timestamp)
elif 'event' not in msg:
LOG.warning('%s: Unexpected msg (missing event) from exchange: %s',
conn.uuid, msg)
elif msg['event'] == 'error':
LOG.error('%s: Error from exchange: %s', conn.uuid, msg)
elif msg['event'] in ('info', 'conf'):
LOG.debug('%s: %s from exchange: %s', conn.uuid, msg['event'], msg)
elif 'chanId' in msg and 'symbol' in msg:
self.register_channel_handler(msg, conn)
else:
LOG.warning('%s: Unexpected msg from exchange: %s', conn.uuid, msg)
def register_channel_handler(self, msg: dict, conn: AsyncConnection):
symbol = msg['symbol']
is_funding = (symbol[0] == 'f')
pair = self.exchange_symbol_to_std_symbol(symbol)
if msg['channel'] == 'ticker':
if is_funding:
LOG.warning(
'%s %s: Ticker funding not implemented - ignoring for %s',
conn.uuid, pair, msg)
handler = self._do_nothing
else:
handler = partial(self._ticker, pair)
elif msg['channel'] == 'trades':
if is_funding:
handler = partial(self._funding, pair)
else:
handler = partial(self._trades, pair)
elif msg['channel'] == 'book':
if msg['prec'] == 'R0':
handler = partial(self._raw_book, pair)
elif is_funding:
LOG.warning(
'%s %s: Book funding not implemented - ignoring for %s',
conn.uuid, pair, msg)
handler = self._do_nothing
else:
handler = partial(self._book, pair)
else:
LOG.warning('%s %s: Unexpected message %s', conn.uuid, pair, msg)
return
LOG.debug(
'%s: Register channel=%s pair=%s funding=%s %s -> %s()', conn.uuid,
msg['channel'], pair, is_funding,
'='.join(list(msg.items())[-1]), handler.__name__ if hasattr(
handler, '__name__') else handler.func.__name__)
self.handlers[msg['chanId']] = handler
async def subscribe(self, connection: AsyncConnection):
self.__reset(connection)
await connection.write(json.dumps({'event': "conf", 'flags': SEQ_ALL}))
for chan, pairs in connection.subscription.items():
for pair in pairs:
message = {
'event': 'subscribe',
'channel': chan,
'symbol': pair
}
if 'book' in chan:
parts = chan.split('-')
if len(parts) != 1:
message['channel'] = 'book'
try:
message['prec'] = parts[1]
message['freq'] = self.book_frequency
message['len'] = self.number_of_price_points
except IndexError:
# any non specified params will be defaulted
pass
await connection.write(json.dumps(message))
class Bitflyer(Feed):
id = BITFLYER
websocket_endpoints = [
WebsocketEndpoint('wss://ws.lightstream.bitflyer.com/json-rpc')
]
rest_endpoints = [
RestEndpoint('https://api.bitflyer.com',
routes=Routes([
'/v1/getmarkets/eu', '/v1/getmarkets/usa',
'/v1/getmarkets', '/v1/markets', '/v1/markets/usa',
'/v1/markets/eu'
]))
]
websocket_channels = {
L2_BOOK: 'lightning_board_{}',
TRADES: 'lightning_executions_{}',
TICKER: 'lightning_ticker_{}'
}
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
for datum in entry:
stype = datum['market_type'].lower()
expiration = None
if stype == FUTURES:
base, quote = datum['product_code'][:3], datum[
'product_code'][3:6]
expiration = datum['product_code'][6:]
elif stype == FX:
_, base, quote = datum['product_code'].split("_")
else:
base, quote = datum['product_code'].split("_")
s = Symbol(base, quote, type=stype, expiry_date=expiration)
ret[s.normalized] = datum['product_code']
info['instrument_type'][s.normalized] = stype
return ret, info
def __reset(self):
self._l2_book = {}
async def _ticker(self, msg: dict, timestamp: float):
"""
{
"jsonrpc": "2.0",
"method": "channelMessage",
"params": {
"channel": "lightning_ticker_BTC_USD",
"message": {
"product_code": "BTC_USD",
"state": "RUNNING",
"timestamp":"2020-12-25T21:16:19.3661298Z",
"tick_id": 703768,
"best_bid": 24228.97,
"best_ask": 24252.89,
"best_bid_size": 0.4006,
"best_ask_size": 0.4006,
"total_bid_depth": 64.73938803,
"total_ask_depth": 51.99613815,
"market_bid_size": 0.0,
"market_ask_size": 0.0,
"ltp": 24382.25,
"volume": 241.953371650000,
"volume_by_product": 241.953371650000
}
}
}
"""
pair = self.exchange_symbol_to_std_symbol(
msg['params']['message']['product_code'])
bid = msg['params']['message']['best_bid']
ask = msg['params']['message']['best_ask']
t = Ticker(self.id,
pair,
bid,
ask,
self.timestamp_normalize(
msg['params']['message']['timestamp']),
raw=msg)
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{
"jsonrpc":"2.0",
"method":"channelMessage",
"params":{
"channel":"lightning_executions_BTC_JPY",
"message":[
{
"id":2084881071,
"side":"BUY",
"price":2509125.0,
"size":0.005,
"exec_date":"2020-12-25T21:36:22.8840579Z",
"buy_child_order_acceptance_id":"JRF20201225-213620-004123",
"sell_child_order_acceptance_id":"JRF20201225-213620-133314"
}
]
}
}
"""
pair = self.exchange_symbol_to_std_symbol(
msg['params']['channel'][21:])
for update in msg['params']['message']:
t = Trade(self.id,
pair,
BUY if update['side'] == 'BUY' else SELL,
update['size'],
update['price'],
self.timestamp_normalize(update['exec_date']),
raw=update)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
"""
{
"jsonrpc":"2.0",
"method":"channelMessage",
"params":{
"channel":"lightning_board_BTC_JPY",
"message":{
"mid_price":2534243.0,
"bids":[
],
"asks":[
{
"price":2534500.0,
"size":0.0
},
{
"price":2536101.0,
"size":0.0
}
]
}
}
}
"""
snapshot = msg['params']['channel'].startswith(
'lightning_board_snapshot')
if snapshot:
pair = msg['params']['channel'].split(
"lightning_board_snapshot")[1][1:]
else:
pair = msg['params']['channel'].split("lightning_board")[1][1:]
pair = self.exchange_symbol_to_std_symbol(pair)
# Ignore deltas until a snapshot is received
if pair not in self._l2_book and not snapshot:
return
delta = None
if snapshot:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
else:
delta = {BID: [], ASK: []}
data = msg['params']['message']
for side in ('bids', 'asks'):
s = BID if side == 'bids' else ASK
if snapshot:
self._l2_book[pair].book[side] = {
d['price']: d['size']
for d in data[side]
}
else:
for entry in data[side]:
if entry['size'] == 0:
if entry['price'] in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][entry['price']]
delta[s].append((entry['price'], Decimal(0.0)))
else:
self._l2_book[pair].book[side][
entry['price']] = entry['size']
delta[s].append((entry['price'], entry['size']))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
delta=delta)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['params']['channel'].startswith("lightning_ticker_"):
await self._ticker(msg, timestamp)
elif msg['params']['channel'].startswith('lightning_executions_'):
await self._trade(msg, timestamp)
elif msg['params']['channel'].startswith('lightning_board_'):
await self._book(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
for pair in self.subscription[chan]:
if chan.startswith('lightning_board'):
# need to subscribe to snapshots too if subscribed to L2_BOOKS
await conn.write(
json.dumps({
"method": "subscribe",
"params": {
"channel": f'lightning_board_snapshot_{pair}'
}
}))
await conn.write(
json.dumps({
"method": "subscribe",
"params": {
"channel": chan.format(pair)
}
}))
class Delta(Feed):
id = DELTA
websocket_endpoints = [
WebsocketEndpoint('wss://socket.delta.exchange',
sandbox='wss://testnet-socket.delta.exchange')
]
rest_endpoints = [
RestEndpoint('https://api.delta.exchange',
routes=Routes('/v2/products'))
]
websocket_channels = {
L2_BOOK: 'l2_orderbook',
TRADES: 'all_trades',
CANDLES: 'candlestick_',
}
valid_candle_intervals = {
'1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '12h', '1d',
'1w', '2w', '1M'
}
candle_interval_map = {
'1m': '1m',
'3m': '3m',
'5m': '5m',
'15m': '15m',
'30m': '30m',
'1h': '1h',
'2h': '2h',
'4h': '4h',
'6h': '6h',
'12h': '12h',
'1d': '1d',
'1w': '1w',
'2w': '2w',
'1M': '30d'
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1_000_000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data['result']:
quote = entry['quoting_asset']['symbol']
base = entry['underlying_asset']['symbol']
if entry['contract_type'] == 'spot':
sym = Symbol(base, quote, type=SPOT)
elif entry['contract_type'] == 'perpetual_futures':
sym = Symbol(base, quote, type=PERPETUAL)
elif entry['contract_type'] == 'futures' or entry[
'contract_type'] == 'move_options':
sym = Symbol(base,
quote,
type=FUTURES,
expiry_date=entry['settlement_time'])
elif entry['contract_type'] == 'call_options' or entry[
'contract_type'] == 'put_options':
otype = PUT if entry['contract_type'].startswith(
'put') else CALL
sym = Symbol(base,
quote,
type=OPTION,
strike_price=entry['strike_price'],
expiry_date=entry['settlement_time'],
option_type=otype)
elif entry['contract_type'] in {'interest_rate_swaps', 'spreads'}:
continue
else:
raise ValueError(entry['contract_type'])
info['instrument_type'][sym.normalized] = sym.type
info['tick_size'][sym.normalized] = entry['tick_size']
ret[sym.normalized] = entry['symbol']
return ret, info
def __reset(self):
self._l2_book = {}
async def _trades(self, msg: dict, timestamp: float):
'''
{
'buyer_role': 'taker',
'price': '54900.0',
'product_id': 8320,
'seller_role': 'maker',
'size': '0.000695',
'symbol': 'BTC_USDT',
'timestamp': 1638132618257226,
'type': 'all_trades'
}
'''
if msg['type'] == 'all_trades':
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
BUY if msg['buyer_role'] == 'taker' else SELL,
Decimal(msg['size']),
Decimal(msg['price']),
self.timestamp_normalize(msg['timestamp']),
raw=msg)
await self.callback(TRADES, t, timestamp)
else:
for trade in msg['trades']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
BUY if trade['buyer_role'] == 'taker' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
self.timestamp_normalize(trade['timestamp']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _candles(self, msg: dict, timestamp: float):
'''
{
'candle_start_time': 1638134700000000,
'close': None,
'high': None,
'last_updated': 1638134700318213,
'low': None,
'open': None,
'resolution': '1m',
'symbol': 'BTC_USDT',
'timestamp': 1638134708903082,
'type': 'candlestick_1m',
'volume': 0
}
'''
interval = self.normalize_candle_interval[msg['resolution']]
c = Candle(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
self.timestamp_normalize(msg['candle_start_time']),
self.timestamp_normalize(msg['candle_start_time']) +
timedelta_str_to_sec(interval) - 1,
interval,
None,
Decimal(msg['open'] if msg['open'] else 0),
Decimal(msg['close'] if msg['close'] else 0),
Decimal(msg['high'] if msg['high'] else 0),
Decimal(msg['low'] if msg['low'] else 0),
Decimal(msg['volume'] if msg['volume'] else 0),
False,
self.timestamp_normalize(msg['timestamp']),
raw=msg)
await self.callback(CANDLES, c, timestamp)
async def _book(self, msg: dict, timestamp: float):
'''
{
'buy': [
{
'depth': '0.007755',
'limit_price': '55895.5',
'size': '0.007755'
},
...
],
'last_sequence_no': 1638135705586546,
'last_updated_at': 1638135705559000,
'product_id': 8320,
'sell': [
{
'depth': '0.008855',
'limit_price': '55901.5',
'size': '0.008855'
},
...
],
'symbol': 'BTC_USDT',
'timestamp': 1638135705586546,
'type': 'l2_orderbook'
}
'''
symbol = self.exchange_symbol_to_std_symbol(msg['symbol'])
if symbol not in self._l2_book:
self._l2_book[symbol] = OrderBook(self.id,
symbol,
max_depth=self.max_depth)
self._l2_book[symbol].book.bids = {
Decimal(e['limit_price']): Decimal(e['size'])
for e in msg['buy']
}
self._l2_book[symbol].book.asks = {
Decimal(e['limit_price']): Decimal(e['size'])
for e in msg['sell']
}
await self.book_callback(L2_BOOK,
self._l2_book[symbol],
timestamp,
timestamp=self.timestamp_normalize(
msg['timestamp']),
raw=msg)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['type'] == 'l2_orderbook':
await self._book(msg, timestamp)
elif msg['type'].startswith('all_trades'):
await self._trades(msg, timestamp)
elif msg['type'].startswith('candlestick'):
await self._candles(msg, timestamp)
elif msg['type'] == 'subscriptions':
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
await conn.write(
json.dumps({
"type": "subscribe",
"payload": {
"channels": [{
"name":
c if c != 'candlestick_' else c +
self.candle_interval_map[self.candle_interval],
"symbols":
list(self.subscription[c])
} for c in self.subscription]
}
}))
class Huobi(Feed):
id = HUOBI
websocket_endpoints = [WebsocketEndpoint('wss://api.huobi.pro/ws')]
rest_endpoints = [
RestEndpoint('https://api.huobi.pro',
routes=Routes('/v1/common/symbols'))
]
valid_candle_intervals = {
'1m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '1M', '1Y'
}
candle_interval_map = {
'1m': '1min',
'5m': '5min',
'15m': '15min',
'30m': '30min',
'1h': '60min',
'4h': '4hour',
'1d': '1day',
'1M': '1mon',
'1w': '1week',
'1Y': '1year'
}
websocket_channels = {
L2_BOOK: 'depth.step0',
TRADES: 'trade.detail',
CANDLES: 'kline',
TICKER: 'ticker'
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
for e in data['data']:
if e['state'] == 'offline':
continue
base, quote = e['base-currency'].upper(
), e['quote-currency'].upper()
s = Symbol(base, quote)
ret[s.normalized] = e['symbol']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
async def _book(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1])
data = msg['tick']
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for price, amount in data['asks']
}
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(msg['ts']),
raw=msg)
async def _ticker(self, msg: dict, timestamp: float):
"""
{
"ch":"market.btcusdt.ticker",
"ts":1630982370526,
"tick":{
"open":51732,
"high":52785.64,
"low":51000,
"close":52735.63,
"amount":13259.24137056181,
"vol":687640987.4125315,
"count":448737,
"bid":52732.88,
"bidSize":0.036,
"ask":52732.89,
"askSize":0.583653,
"lastPrice":52735.63,
"lastSize":0.03
}
}
"""
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1]),
msg['tick']['bid'],
msg['tick']['ask'],
self.timestamp_normalize(msg['ts']),
raw=msg['tick'])
await self.callback(TICKER, t, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'ch': 'market.adausdt.trade.detail',
'ts': 1597792835344,
'tick': {
'id': 101801945127,
'ts': 1597792835336,
'data': [
{
'id': Decimal('10180194512782291967181675'), <- per docs this is deprecated
'ts': 1597792835336,
'tradeId': 100341530602,
'amount': Decimal('0.1'),
'price': Decimal('0.137031'),
'direction': 'sell'
}
]
}
}
"""
for trade in msg['tick']['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(
msg['ch'].split('.')[1]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
self.timestamp_normalize(trade['ts']),
id=str(trade['tradeId']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _candles(self, msg: dict, symbol: str, interval: str,
timestamp: float):
"""
{
'ch': 'market.btcusdt.kline.1min',
'ts': 1618700872863,
'tick': {
'id': 1618700820,
'open': Decimal('60751.62'),
'close': Decimal('60724.73'),
'low': Decimal('60724.73'),
'high': Decimal('60751.62'),
'amount': Decimal('2.1990737759143966'),
'vol': Decimal('133570.944386'),
'count': 235}
}
}
"""
interval = self.normalize_candle_interval[interval]
start = int(msg['tick']['id'])
end = start + timedelta_str_to_sec(interval) - 1
c = Candle(self.id,
self.exchange_symbol_to_std_symbol(symbol),
start,
end,
interval,
msg['tick']['count'],
Decimal(msg['tick']['open']),
Decimal(msg['tick']['close']),
Decimal(msg['tick']['high']),
Decimal(msg['tick']['low']),
Decimal(msg['tick']['amount']),
None,
self.timestamp_normalize(msg['ts']),
raw=msg)
await self.callback(CANDLES, c, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
# unzip message
msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS)
msg = json.loads(msg, parse_float=Decimal)
# Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it
if 'ping' in msg:
await conn.write(json.dumps({'pong': msg['ping']}))
elif 'status' in msg and msg['status'] == 'ok':
return
elif 'ch' in msg:
if 'trade' in msg['ch']:
await self._trade(msg, timestamp)
elif 'tick' in msg['ch']:
await self._ticker(msg, timestamp)
elif 'depth' in msg['ch']:
await self._book(msg, timestamp)
elif 'kline' in msg['ch']:
_, symbol, _, interval = msg['ch'].split(".")
await self._candles(msg, symbol, interval, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
client_id = 0
for chan in self.subscription:
for pair in self.subscription[chan]:
client_id += 1
normalized_chan = self.exchange_channel_to_std(chan)
await conn.write(
json.dumps({
"sub":
f"market.{pair}.{chan}"
if normalized_chan != CANDLES else
f"market.{pair}.{chan}.{self.candle_interval_map[self.candle_interval]}",
"id":
client_id
}))
class AscendEX(Feed):
id = ASCENDEX
rest_endpoints = [
RestEndpoint('https://ascendex.com',
routes=Routes('/api/pro/v1/products'),
sandbox='https://api-test.ascendex-sandbox.com')
]
websocket_channels = {
L2_BOOK: 'depth:',
TRADES: 'trades:',
}
# Docs, https://ascendex.github.io/ascendex-pro-api/#websocket-authentication
# noinspection PyTypeChecker
websocket_endpoints = [
WebsocketEndpoint(
'wss://ascendex.com/1/api/pro/v1/stream',
channel_filter=(
websocket_channels[L2_BOOK],
websocket_channels[TRADES],
),
sandbox='wss://api-test.ascendex-sandbox.com/1/api/pro/v1/stream',
)
]
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data['data']:
# Only "Normal" status symbols are tradeable
if entry['status'] == 'Normal':
s = Symbol(entry['baseAsset'], entry['quoteAsset'])
ret[s.normalized] = entry['symbol']
info['tick_size'][s.normalized] = entry['tickSize']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
self.seq_no = defaultdict(lambda: None)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'm': 'trades',
'symbol': 'BTC/USDT',
'data': [{
'p': '23169.76',
'q': '0.00899',
'ts': 1608760026461,
'bm': False,
'seqnum': 72057614186183012
}]
}
"""
for trade in msg['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(msg['symbol']),
SELL if trade['bm'] else BUY,
Decimal(trade['q']),
Decimal(trade['p']),
self.timestamp_normalize(trade['ts']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
sequence_number = msg['data']['seqnum']
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
delta = {BID: [], ASK: []}
if msg['m'] == 'depth-snapshot':
self.seq_no[pair] = sequence_number
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
else:
# ignore messages while we wait for the snapshot
if self.seq_no[pair] is None:
return
if self.seq_no[pair] + 1 != sequence_number:
raise MissingSequenceNumber
self.seq_no[pair] = sequence_number
for side in ('bids', 'asks'):
for price, amount in msg['data'][side]:
s = BID if side == 'bids' else ASK
price = Decimal(price)
size = Decimal(amount)
if size == 0:
delta[s].append((price, 0))
if price in self._l2_book[pair].book[s]:
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, size))
self._l2_book[pair].book[s][price] = size
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(msg['data']['ts']),
raw=msg,
delta=delta if msg['m'] != 'depth-snapshot' else None,
sequence_number=sequence_number)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'm' in msg:
if msg['m'] == 'depth' or msg['m'] == 'depth-snapshot':
await self._book(msg, timestamp)
elif msg['m'] == 'trades':
await self._trade(msg, timestamp)
elif msg['m'] == 'ping':
await conn.write('{"op":"pong"}')
elif msg['m'] == 'connected':
return
elif msg['m'] == 'sub':
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
l2_pairs = []
for channel in self.subscription:
pairs = self.subscription[channel]
if channel == "depth:":
l2_pairs.extend(pairs)
message = {'op': 'sub', 'ch': channel + ','.join(pairs)}
await conn.write(json.dumps(message))
for pair in l2_pairs:
message = {
"op": "req",
"action": "depth-snapshot",
"args": {
"symbol": pair
}
}
await conn.write(json.dumps(message))
class Deribit(Feed, DeribitRestMixin):
id = DERIBIT
websocket_endpoints = [
WebsocketEndpoint('wss://www.deribit.com/ws/api/v2',
sandbox='wss://test.deribit.com/ws/api/v2')
]
rest_endpoints = [
RestEndpoint('https://www.deribit.com',
sandbox='https://test.deribit.com',
routes=Routes([
'/api/v2/public/get_instruments?currency=BTC',
'/api/v2/public/get_instruments?currency=ETH',
'/api/v2/public/get_instruments?currency=USDC',
'/api/v2/public/get_instruments?currency=SOL'
]))
]
websocket_channels = {
L1_BOOK: 'quote',
L2_BOOK: 'book',
TRADES: 'trades',
TICKER: 'ticker',
FUNDING: 'ticker',
OPEN_INTEREST: 'ticker',
LIQUIDATIONS: 'trades',
ORDER_INFO: 'user.orders',
FILLS: 'user.trades',
BALANCES: 'user.portfolio',
}
request_limit = 20
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def is_authenticated_channel(cls, channel: str) -> bool:
return channel in (ORDER_INFO, FILLS, BALANCES, L2_BOOK, TRADES,
TICKER)
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
currencies = []
for entry in data:
for e in entry['result']:
base = e['base_currency']
if base not in currencies:
currencies.append(base)
quote = e['quote_currency']
stype = e['kind'] if e[
'settlement_period'] != 'perpetual' else PERPETUAL
otype = e.get('option_type')
strike_price = e.get('strike')
strike_price = int(strike_price) if strike_price else None
expiry = e['expiration_timestamp'] / 1000
s = Symbol(base,
quote,
type=FUTURES if stype == 'future' else stype,
option_type=otype,
strike_price=strike_price,
expiry_date=expiry)
ret[s.normalized] = e['instrument_name']
info['tick_size'][s.normalized] = e['tick_size']
info['instrument_type'][s.normalized] = stype
for currency in currencies:
s = Symbol(currency, currency, type=CURRENCY)
ret[s.normalized] = currency
info['instrument_type'][s.normalized] = CURRENCY
return ret, info
def __reset(self):
self._open_interest_cache = {}
self._l2_book = {}
self.seq_no = {}
async def _trade(self, msg: dict, timestamp: float):
"""
{
"params":
{
"data":
[
{
"trade_seq": 933,
"trade_id": "9178",
"timestamp": 1550736299753,
"tick_direction": 3,
"price": 3948.69,
"instrument_name": "BTC-PERPETUAL",
"index_price": 3930.73,
"direction": "sell",
"amount": 10
}
],
"channel": "trades.BTC-PERPETUAL.raw"
},
"method": "subscription",
"jsonrpc": "2.0"
}
"""
for trade in msg["params"]["data"]:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(
trade["instrument_name"]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
self.timestamp_normalize(trade['timestamp']),
id=trade['trade_id'],
raw=trade)
await self.callback(TRADES, t, timestamp)
if 'liquidation' in trade:
liq = Liquidation(self.id,
self.exchange_symbol_to_std_symbol(
trade["instrument_name"]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
trade['trade_id'],
FILLED,
self.timestamp_normalize(trade['timestamp']),
raw=trade)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def _ticker(self, msg: dict, timestamp: float):
'''
{
"params" : {
"data" : {
"timestamp" : 1550652954406,
"stats" : {
"volume" : null,
"low" : null,
"high" : null
},
"state" : "open",
"settlement_price" : 3960.14,
"open_interest" : 0.12759952124659626,
"min_price" : 3943.21,
"max_price" : 3982.84,
"mark_price" : 3940.06,
"last_price" : 3906,
"instrument_name" : "BTC-PERPETUAL",
"index_price" : 3918.51,
"funding_8h" : 0.01520525,
"current_funding" : 0.00499954,
"best_bid_price" : 3914.97,
"best_bid_amount" : 40,
"best_ask_price" : 3996.61,
"best_ask_amount" : 50
},
"channel" : "ticker.BTC-PERPETUAL.raw"
},
"method" : "subscription",
"jsonrpc" : "2.0"}
'''
pair = self.exchange_symbol_to_std_symbol(
msg['params']['data']['instrument_name'])
ts = self.timestamp_normalize(msg['params']['data']['timestamp'])
t = Ticker(self.id,
pair,
Decimal(msg["params"]["data"]['best_bid_price']),
Decimal(msg["params"]["data"]['best_ask_price']),
ts,
raw=msg)
await self.callback(TICKER, t, timestamp)
if "current_funding" in msg["params"]["data"] and "funding_8h" in msg[
"params"]["data"]:
f = Funding(self.id,
pair,
Decimal(msg['params']['data']['mark_price']),
Decimal(msg["params"]["data"]["current_funding"]),
None,
ts,
raw=msg)
await self.callback(FUNDING, f, timestamp)
oi = msg['params']['data']['open_interest']
if pair in self._open_interest_cache and oi == self._open_interest_cache[
pair]:
return
self._open_interest_cache[pair] = oi
o = OpenInterest(self.id, pair, Decimal(oi), ts, raw=msg)
await self.callback(OPEN_INTEREST, o, timestamp)
async def _quote(self, quote: dict, timestamp: float):
book = L1Book(self.id,
self.exchange_symbol_to_std_symbol(
quote['instrument_name']),
Decimal(quote['best_bid_price']),
Decimal(quote['best_bid_amount']),
Decimal(quote['best_ask_price']),
Decimal(quote['best_ask_amount']),
self.timestamp_normalize(quote['timestamp']),
raw=quote)
await self.callback(L1_BOOK, book, timestamp)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
pub_channels = []
pri_channels = []
for chan in self.subscription:
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
for pair in self.subscription[chan]:
if self.exchange_channel_to_std(chan) == BALANCES:
pri_channels.append(f"{chan}.{pair}")
else:
pri_channels.append(f"{chan}.{pair}.raw")
else:
for pair in self.subscription[chan]:
if self.exchange_channel_to_std(chan) == L1_BOOK:
pub_channels.append(f"{chan}.{pair}")
else:
pub_channels.append(f"{chan}.{pair}.raw")
if pub_channels:
msg = {
"jsonrpc": "2.0",
"id": "101",
"method": "public/subscribe",
"params": {
"channels": pub_channels
}
}
LOG.debug(
f'{conn.uuid}: Subscribing to public channels with message {msg}'
)
await conn.write(json.dumps(msg))
if pri_channels:
msg = {
"jsonrpc": "2.0",
"id": "102",
"method": "private/subscribe",
"params": {
"scope": f"session:{conn.uuid}",
"channels": pri_channels
}
}
LOG.debug(
f'{conn.uuid}: Subscribing to private channels with message {msg}'
)
await conn.write(json.dumps(msg))
async def _book_snapshot(self, msg: dict, timestamp: float):
"""
{
'jsonrpc': '2.0',
'method': 'subscription',
'params': {
'channel': 'book.BTC-PERPETUAL.raw',
'data': {
'timestamp': 1598232105378,
'instrument_name': 'BTC-PERPETUAL',
'change_id': 21486665526, '
'bids': [['new', Decimal('11618.5'), Decimal('279310.0')], ..... ]
'asks': [[ ....... ]]
}
}
}
"""
ts = msg["params"]["data"]["timestamp"]
pair = self.exchange_symbol_to_std_symbol(
msg["params"]["data"]["instrument_name"])
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for _, price, amount in msg["params"]["data"]["bids"]
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for _, price, amount in msg["params"]["data"]["asks"]
}
self.seq_no[pair] = msg["params"]["data"]["change_id"]
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(ts),
sequence_number=msg["params"]["data"]["change_id"],
raw=msg)
async def _book_update(self, msg: dict, timestamp: float):
ts = msg["params"]["data"]["timestamp"]
pair = self.exchange_symbol_to_std_symbol(
msg["params"]["data"]["instrument_name"])
if msg['params']['data']['prev_change_id'] != self.seq_no[pair]:
LOG.warning("%s: Missing sequence number detected for %s", self.id,
pair)
LOG.warning("%s: Requesting book snapshot", self.id)
raise MissingSequenceNumber
self.seq_no[pair] = msg['params']['data']['change_id']
delta = {BID: [], ASK: []}
for action, price, amount in msg["params"]["data"]["bids"]:
if action != "delete":
self._l2_book[pair].book.bids[price] = Decimal(amount)
delta[BID].append((Decimal(price), Decimal(amount)))
else:
del self._l2_book[pair].book.bids[price]
delta[BID].append((Decimal(price), Decimal(amount)))
for action, price, amount in msg["params"]["data"]["asks"]:
if action != "delete":
self._l2_book[pair].book.asks[price] = amount
delta[ASK].append((Decimal(price), Decimal(amount)))
else:
del self._l2_book[pair].book.asks[price]
delta[ASK].append((Decimal(price), Decimal(amount)))
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(ts),
raw=msg,
delta=delta,
sequence_number=msg['params']['data']['change_id'])
async def message_handler(self, msg: str, conn, timestamp: float):
msg_dict = json.loads(msg, parse_float=Decimal)
if 'error' in msg_dict.keys():
LOG.error("%s: Received Error message: %s, Error code: %s",
conn.uuid, msg_dict['error']['message'],
msg_dict['error']['code'])
elif "result" in msg_dict:
result = msg_dict["result"]
if 'id' in msg_dict:
id = str(msg_dict["id"])
if id == "0":
LOG.debug("%s: Connected", conn.uuid)
elif id == '101':
LOG.info("%s: Subscribed to public channels", conn.uuid)
LOG.debug("%s: %s", conn.uuid, result)
elif id == '102':
LOG.info("%s: Subscribed to authenticated channels",
conn.uuid)
LOG.debug("%s: %s", conn.uuid, result)
elif id.startswith('auth') and "access_token" in result:
'''
Access token is another way to be authenticated while sending messages to Deribit.
In this implementation 'scope session' method is used instead of 'acces token' method.
'''
LOG.debug(f"{conn.uuid}: Access token received")
else:
LOG.warning("%s: Unknown id in message %s", conn.uuid,
msg_dict)
else:
LOG.warning("%s: Unknown 'result' message received: %s",
conn.uuid, msg_dict)
elif 'params' in msg_dict:
params = msg_dict['params']
if 'channel' in params:
channel = params['channel']
if "ticker" == channel.split(".")[0]:
await self._ticker(msg_dict, timestamp)
elif "trades" == channel.split(".")[0]:
await self._trade(msg_dict, timestamp)
elif "book" == channel.split(".")[0]:
# checking if we got full book or its update
# if it's update there is 'prev_change_id' field
if "prev_change_id" not in msg_dict["params"]["data"].keys(
):
await self._book_snapshot(msg_dict, timestamp)
elif "prev_change_id" in msg_dict["params"]["data"].keys():
await self._book_update(msg_dict, timestamp)
elif "quote" == channel.split(".")[0]:
await self._quote(params['data'], timestamp)
elif channel.startswith("user"):
await self._user_channels(conn, msg_dict, timestamp,
channel.split(".")[1])
else:
LOG.warning("%s: Unknown channel %s", conn.uuid, msg_dict)
else:
LOG.warning("%s: Unknown 'params' message received: %s",
conn.uuid, msg_dict)
else:
LOG.warning("%s: Unknown message in msg_dict: %s", conn.uuid,
msg_dict)
async def authenticate(self, conn: AsyncConnection):
if self.requires_authentication:
auth = self._auth(self.key_id, self.key_secret, conn.uuid)
LOG.debug(f"{conn.uuid}: Authenticating with message: {auth}")
await conn.write(json.dumps(auth))
def _auth(self, key_id, key_secret, session_id: str) -> str:
# https://docs.deribit.com/?python#authentication
timestamp = round(datetime.now().timestamp() * 1000)
nonce = f'xyz{str(timestamp)[-5:]}'
signature = hmac.new(bytes(key_secret, "latin-1"),
bytes('{}\n{}\n{}'.format(timestamp, nonce, ""),
"latin-1"),
digestmod=hashlib.sha256).hexdigest().lower()
auth = {
"jsonrpc": "2.0",
"id": f"auth_{session_id}",
"method": "public/auth",
"params": {
"grant_type": "client_signature",
"client_id": key_id,
"timestamp": timestamp,
"signature": signature,
"nonce": nonce,
"data": "",
"scope": f"session:{session_id}"
}
}
return auth
async def _user_channels(self, conn: AsyncConnection, msg: dict,
timestamp: float, subchan: str):
order_status = {
"open": OPEN,
"filled": FILLED,
"rejected": FAILED,
"cancelled": CANCELLED,
"untriggered": OPEN
}
order_types = {
"limit": LIMIT,
"market": MARKET,
"stop_limit": STOP_LIMIT,
"stop_market": STOP_MARKET
}
if 'data' in msg['params']:
data = msg['params']['data']
if subchan == 'portfolio':
'''
{
"params" : {
"data" : {
"total_pl" : 0.00000425,
"session_upl" : 0.00000425,
"session_rpl" : -2e-8,
"projected_maintenance_margin" : 0.00009141,
"projected_initial_margin" : 0.00012542,
"projected_delta_total" : 0.0043,
"portfolio_margining_enabled" : false,
"options_vega" : 0,
"options_value" : 0,
"options_theta" : 0,
"options_session_upl" : 0,
"options_session_rpl" : 0,
"options_pl" : 0,
"options_gamma" : 0,
"options_delta" : 0,
"margin_balance" : 0.2340038,
"maintenance_margin" : 0.00009141,
"initial_margin" : 0.00012542,
"futures_session_upl" : 0.00000425,
"futures_session_rpl" : -2e-8,
"futures_pl" : 0.00000425,
"estimated_liquidation_ratio" : 0.01822795,
"equity" : 0.2340038,
"delta_total" : 0.0043,
"currency" : "BTC",
"balance" : 0.23399957,
"available_withdrawal_funds" : 0.23387415,
"available_funds" : 0.23387838
},
"channel" : "user.portfolio.btc"
},
"method" : "subscription",
"jsonrpc" : "2.0"
}
'''
b = Balance(self.id,
data['currency'],
Decimal(data['balance']),
Decimal(data['balance']) -
Decimal(data['available_withdrawal_funds']),
raw=data)
await self.callback(BALANCES, b, timestamp)
elif subchan == 'orders':
'''
{
"params" : {
"data" : {
"time_in_force" : "good_til_cancelled",
"replaced" : false,
"reduce_only" : false,
"profit_loss" : 0,
"price" : 10502.52,
"post_only" : false,
"original_order_type" : "market",
"order_type" : "limit",
"order_state" : "open",
"order_id" : "5",
"max_show" : 200,
"last_update_timestamp" : 1581507423789,
"label" : "",
"is_liquidation" : false,
"instrument_name" : "BTC-PERPETUAL",
"filled_amount" : 0,
"direction" : "buy",
"creation_timestamp" : 1581507423789,
"commission" : 0,
"average_price" : 0,
"api" : false,
"amount" : 200
},
"channel" : "user.orders.BTC-PERPETUAL.raw"
},
"method" : "subscription",
"jsonrpc" : "2.0"
}
'''
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(
data['instrument_name']),
data["order_id"],
BUY if msg["side"] == 'Buy' else SELL,
order_status[data["order_state"]],
order_types[data['order_type']],
Decimal(data['price']),
Decimal(data['filled_amount']),
Decimal(data['amount']) - Decimal(data['cumQuantity']),
self.timestamp_normalize(data["last_update_timestamp"]),
raw=data)
await self.callback(ORDER_INFO, oi, timestamp)
elif subchan == 'trades':
'''
{
"params" : {
"data" : [
{
"trade_seq" : 30289432,
"trade_id" : "48079254",
"timestamp" : 1590484156350,
"tick_direction" : 0,
"state" : "filled",
"self_trade" : false,
"reduce_only" : false,
"price" : 8954,
"post_only" : false,
"order_type" : "market",
"order_id" : "4008965646",
"matching_id" : null,
"mark_price" : 8952.86,
"liquidity" : "T",
"instrument_name" : "BTC-PERPETUAL",
"index_price" : 8956.73,
"fee_currency" : "BTC",
"fee" : 0.00000168,
"direction" : "sell",
"amount" : 20
}]
}
}
'''
for entry in data:
symbol = self.exchange_symbol_to_std_symbol(
entry['instrument_name'])
f = Fill(self.id,
symbol,
SELL if entry['direction'] == 'sell' else BUY,
Decimal(entry['amount']),
Decimal(entry['price']),
Decimal(entry['fee']),
entry['trade_id'],
entry['order_id'],
entry['order_type'],
TAKER if entry['liquidity'] == 'T' else MAKER,
self.timestamp_normalize(entry['timestamp']),
raw=entry)
await self.callback(FILLS, f, timestamp)
else:
LOG.warning("%s: Unknown channel 'user.%s'", conn.uuid,
subchan)
else:
LOG.warning("%s: Unknown message %s'", conn.uuid, msg)
class Kraken(Feed, KrakenRestMixin):
id = KRAKEN
websocket_endpoints = [WebsocketEndpoint('wss://ws.kraken.com', limit=20)]
rest_endpoints = [
RestEndpoint('https://api.kraken.com',
routes=Routes('/0/public/AssetPairs'))
]
valid_candle_intervals = {
'1m', '5m', '15m', '30m', '1h', '4h', '1d', '1w', '15d'
}
candle_interval_map = {
'1m': 1,
'5m': 5,
'15m': 15,
'30m': 30,
'1h': 60,
'4h': 240,
'1d': 1440,
'1w': 10080,
'15d': 21600
}
valid_depths = [10, 25, 100, 500, 1000]
websocket_channels = {
L2_BOOK: 'book',
TRADES: 'trade',
TICKER: 'ticker',
CANDLES: 'ohlc'
}
request_limit = 10
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for symbol in data['result']:
if 'wsname' not in data['result'][symbol] or '.d' in symbol:
# https://blog.kraken.com/post/259/introducing-the-kraken-dark-pool/
# .d is for dark pool symbols
continue
sym = data['result'][symbol]['wsname']
sym = sym.replace('XBT', 'BTC').replace('XDG', 'DOGE')
base, quote = sym.split("/")
s = Symbol(base, quote)
ret[s.normalized] = data['result'][symbol]['wsname']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __init__(self, max_depth=1000, **kwargs):
super().__init__(max_depth=max_depth, **kwargs)
def __reset(self, conn: AsyncConnection):
if self.std_channel_to_exchange(L2_BOOK) in conn.subscription:
for pair in conn.subscription[self.std_channel_to_exchange(
L2_BOOK)]:
std_pair = self.exchange_symbol_to_std_symbol(pair)
if std_pair in self._l2_book:
del self._l2_book[std_pair]
async def subscribe(self, conn: AsyncConnection):
self.__reset(conn)
for chan, symbols in conn.subscription.items():
sub = {"name": chan}
if self.exchange_channel_to_std(chan) == L2_BOOK:
max_depth = self.max_depth if self.max_depth else 1000
if max_depth not in self.valid_depths:
for d in self.valid_depths:
if d > max_depth:
max_depth = d
break
sub['depth'] = max_depth
if self.exchange_channel_to_std(chan) == CANDLES:
sub['interval'] = self.candle_interval_map[
self.candle_interval]
await conn.write(
json.dumps({
"event": "subscribe",
"pair": symbols,
"subscription": sub
}))
async def _trade(self, msg: dict, pair: str, timestamp: float):
"""
example message:
[1,[["3417.20000","0.21222200","1549223326.971661","b","l",""]]]
channel id, price, amount, timestamp, size, limit/market order, misc
"""
for trade in msg[1]:
price, amount, server_timestamp, side, order_type, _ = trade
order_type = 'limit' if order_type == 'l' else 'market'
t = Trade(self.id,
pair,
BUY if side == 'b' else SELL,
Decimal(amount),
Decimal(price),
float(server_timestamp),
type=order_type,
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _ticker(self, msg: dict, pair: str, timestamp: float):
"""
[93, {'a': ['105.85000', 0, '0.46100000'], 'b': ['105.77000', 45, '45.00000000'], 'c': ['105.83000', '5.00000000'], 'v': ['92170.25739498', '121658.17399954'], 'p': ['107.58276', '107.95234'], 't': [4966, 6717], 'l': ['105.03000', '105.03000'], 'h': ['110.33000', '110.33000'], 'o': ['109.45000', '106.78000']}]
channel id, asks: price, wholeLotVol, vol, bids: price, wholeLotVol, close: ...,, vol: ..., VWAP: ..., trades: ..., low: ...., high: ..., open: ...
"""
t = Ticker(self.id,
pair,
Decimal(msg[1]['b'][0]),
Decimal(msg[1]['a'][0]),
None,
raw=msg)
await self.callback(TICKER, t, timestamp)
async def _book(self, msg: dict, pair: str, timestamp: float):
delta = {BID: [], ASK: []}
msg = msg[1:-2]
if 'as' in msg[0]:
# Snapshot
bids = {
Decimal(update[0]): Decimal(update[1])
for update in msg[0]['bs']
}
asks = {
Decimal(update[0]): Decimal(update[1])
for update in msg[0]['as']
}
self._l2_book[pair] = OrderBook(
self.id,
pair,
max_depth=self.max_depth,
bids=bids,
asks=asks,
checksum_format='KRAKEN',
truncate=self.max_depth != self.valid_depths[-1])
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg)
else:
for m in msg:
for s, updates in m.items():
side = False
if s == 'b':
side = BID
elif s == 'a':
side = ASK
if side:
for update in updates:
price, size, *_ = update
price = Decimal(price)
size = Decimal(size)
if size == 0:
# Per Kraken's technical support
# they deliver erroneous deletion messages
# periodically which should be ignored
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
delta[side].append((price, 0))
else:
delta[side].append((price, size))
self._l2_book[pair].book[side][price] = size
if self.checksum_validation and 'c' in msg[0] and self._l2_book[
pair].book.checksum() != int(msg[0]['c']):
raise BadChecksum("Checksum validation on orderbook failed")
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
delta=delta,
raw=msg,
checksum=int(msg[0]['c']) if 'c' in msg[0] else None)
async def _candle(self, msg: list, pair: str, timestamp: float):
"""
[327,
['1621988141.603324', start
'1621988160.000000', end
'38220.70000', open
'38348.80000', high
'38220.70000', low
'38320.40000', close
'38330.59222', vwap
'3.23539643', volume
42 count
],
'ohlc-1',
'XBT/USD']
"""
start, end, open, high, low, close, _, volume, count = msg[1]
interval = int(msg[-2].split("-")[-1])
c = Candle(self.id,
pair,
float(end) - (interval * 60),
float(end),
self.normalize_candle_interval[interval],
count,
Decimal(open),
Decimal(close),
Decimal(high),
Decimal(low),
Decimal(volume),
None,
float(start),
raw=msg)
await self.callback(CANDLES, c, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if isinstance(msg, list):
channel, pair = msg[-2:]
pair = self.exchange_symbol_to_std_symbol(pair)
if channel == 'trade':
await self._trade(msg, pair, timestamp)
elif channel == 'ticker':
await self._ticker(msg, pair, timestamp)
elif channel[:4] == 'book':
await self._book(msg, pair, timestamp)
elif channel[:4] == 'ohlc':
await self._candle(msg, pair, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
if msg['event'] == 'heartbeat':
return
elif msg['event'] == 'systemStatus':
return
elif msg['event'] == 'subscriptionStatus' and msg[
'status'] == 'subscribed':
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
class Probit(Feed):
id = PROBIT
websocket_endpoints = [
WebsocketEndpoint('wss://api.probit.com/api/exchange/v1/ws')
]
rest_endpoints = [
RestEndpoint('https://api.probit.com',
routes=Routes('/api/exchange/v1/market'))
]
websocket_channels = {
L2_BOOK: 'order_books',
TRADES: 'recent_trades',
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
# doc: https://docs-en.probit.com/reference-link/market
for entry in data['data']:
if entry['closed']:
continue
s = Symbol(entry['base_currency_id'], entry['quote_currency_id'])
ret[s.normalized] = entry['id']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self._l2_book = {}
async def _trades(self, msg: dict, timestamp: float):
'''
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok","lag":0,
"recent_trades":[
{
"id":"ETH-BTC:4429182",
"price":"0.028229",
"quantity":"3.117",
"time":"2020-11-01T03:59:06.277Z",
"side":"buy","tick_direction":"down"
},{
"id":"ETH-BTC:4429183",
"price":"0.028227",
"quantity":"1.793",
"time":"2020-11-01T03:59:14.528Z",
"side":"buy",
"tick_direction":"down"
}
],"reset":true
}
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok","lag":0,
"recent_trades":[
{
"id":"ETH-BTC:4429282",
"price":"0.028235",
"quantity":"2.203",
"time":"2020-11-01T04:22:15.117Z",
"side":"buy",
"tick_direction":"down"
}
]
}
'''
pair = self.exchange_symbol_to_std_symbol(msg['market_id'])
for update in msg['recent_trades']:
t = Trade(self.id,
pair,
BUY if update['side'] == 'buy' else SELL,
Decimal(update['quantity']),
Decimal(update['price']),
self.timestamp_normalize(update['time']),
id=update['id'],
raw=update)
await self.callback(TRADES, t, timestamp)
async def _l2_update(self, msg: dict, timestamp: float):
'''
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok",
"lag":0,
"order_books":[
{
"side":"buy",
"price":"0.0165",
"quantity":"0.47"
},{
"side":"buy",
"price":"0",
"quantity":"14656.177"
},{
"side":"sell",
"price":"6400",
"quantity":"0.001"
}],
"reset":true
}
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok",
"lag":0,
"order_books":[
{
"side":"buy",
"price":"0.0281",
"quantity":"48.541"
},{
"side":"sell",
"price":"0.0283",
"quantity":"0"
}]
}
'''
pair = self.exchange_symbol_to_std_symbol(msg['market_id'])
is_snapshot = msg.get('reset', False)
if is_snapshot:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
for entry in msg["order_books"]:
price = Decimal(entry['price'])
quantity = Decimal(entry['quantity'])
side = BID if entry['side'] == "buy" else ASK
self._l2_book[pair].book[side][price] = quantity
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg)
else:
delta = {BID: [], ASK: []}
for entry in msg["order_books"]:
price = Decimal(entry['price'])
quantity = Decimal(entry['quantity'])
side = BID if entry['side'] == "buy" else ASK
if quantity == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
delta[side].append((price, 0))
else:
self._l2_book[pair].book[side][price] = quantity
delta[side].append((price, quantity))
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
delta=delta)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Probit can send multiple type updates in one message so we avoid the use of elif
if 'recent_trades' in msg:
await self._trades(msg, timestamp)
if 'order_books' in msg:
await self._l2_update(msg, timestamp)
# Probit has a 'ticker' channel, but it provide OHLC-last data, not BBO px.
async def subscribe(self, conn: AsyncConnection):
self.__reset()
if self.subscription:
for chan in self.subscription:
for pair in self.subscription[chan]:
await conn.write(
json.dumps({
"type": "subscribe",
"channel": "marketdata",
"filter": [chan],
"interval": 100,
"market_id": pair,
}))
class KrakenFutures(Feed):
id = KRAKEN_FUTURES
websocket_endpoints = [WebsocketEndpoint('wss://futures.kraken.com/ws/v1')]
rest_endpoints = [
RestEndpoint('https://futures.kraken.com',
routes=Routes('/derivatives/api/v3/instruments'))
]
websocket_channels = {
L2_BOOK: 'book',
TRADES: 'trade',
TICKER: 'ticker_lite',
FUNDING: 'ticker',
OPEN_INTEREST: 'ticker',
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
_kraken_futures_product_type = {
'FI': 'Inverse Futures',
'FV': 'Vanilla Futures',
'PI': 'Perpetual Inverse Futures',
'PF': 'Perpetual Linear Multi-Collateral Futures',
'PV': 'Perpetual Vanilla Futures',
'IN': 'Real Time Index',
'RR': 'Reference Rate',
}
ret = {}
info = defaultdict(dict)
data = data['instruments']
for entry in data:
if not entry['tradeable']:
continue
ftype, symbol = entry['symbol'].upper().split("_", maxsplit=1)
stype = PERPETUAL
expiry = None
if "_" in symbol:
stype = FUTURES
symbol, expiry = symbol.split("_")
symbol = symbol.replace('XBT', 'BTC')
base, quote = symbol[:3], symbol[3:]
s = Symbol(base, quote, type=stype, expiry_date=expiry)
info['tick_size'][s.normalized] = entry['tickSize']
info['contract_size'][s.normalized] = entry['contractSize']
info['underlying'][s.normalized] = entry.get('underlying')
info['product_type'][
s.normalized] = _kraken_futures_product_type[ftype]
info['instrument_type'][s.normalized] = stype
ret[s.normalized] = entry['symbol']
return ret, info
def __reset(self):
self._open_interest_cache = {}
self._l2_book = {}
self.seq_no = {}
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
await conn.write(
json.dumps({
"event": "subscribe",
"feed": chan,
"product_ids": self.subscription[chan]
}))
async def _trade(self, msg: dict, pair: str, timestamp: float):
"""
{
"feed": "trade",
"product_id": "PI_XBTUSD",
"uid": "b5a1c239-7987-4207-96bf-02355a3263cf",
"side": "sell",
"type": "fill",
"seq": 85423,
"time": 1565342712903,
"qty": 1135.0,
"price": 11735.0
}
"""
t = Trade(self.id,
pair,
BUY if msg['side'] == 'buy' else SELL,
Decimal(msg['qty']),
Decimal(msg['price']),
self.timestamp_normalize(msg['time']),
id=msg['uid'],
raw=msg)
await self.callback(TRADES, t, timestamp)
async def _ticker(self, msg: dict, pair: str, timestamp: float):
"""
{
"feed": "ticker_lite",
"product_id": "PI_XBTUSD",
"bid": 11726.5,
"ask": 11732.5,
"change": 0.0,
"premium": -0.1,
"volume": "7.0541503E7",
"tag": "perpetual",
"pair": "XBT:USD",
"dtm": -18117,
"maturityTime": 0
}
"""
t = Ticker(self.id, pair, msg['bid'], msg['ask'], None, raw=msg)
await self.callback(TICKER, t, timestamp)
async def _book_snapshot(self, msg: dict, pair: str, timestamp: float):
"""
{
"feed": "book_snapshot",
"product_id": "PI_XBTUSD",
"timestamp": 1565342712774,
"seq": 30007298,
"bids": [
{
"price": 11735.0,
"qty": 50000.0
},
...
],
"asks": [
{
"price": 11739.0,
"qty": 47410.0
},
...
],
"tickSize": null
}
"""
bids = {
Decimal(update['price']): Decimal(update['qty'])
for update in msg['bids']
}
asks = {
Decimal(update['price']): Decimal(update['qty'])
for update in msg['asks']
}
if pair in self._l2_book:
self._l2_book[pair].book.bids = bids
self._l2_book[pair].book.asks = asks
else:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth,
bids=bids,
asks=asks)
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
sequence_number=msg['seq'])
async def _book(self, msg: dict, pair: str, timestamp: float):
"""
Message is received for every book update:
{
"feed": "book",
"product_id": "PI_XBTUSD",
"side": "sell",
"seq": 30007489,
"price": 11741.5,
"qty": 10000.0,
"timestamp": 1565342713929
}
"""
if pair in self.seq_no and self.seq_no[pair] + 1 != msg['seq']:
raise MissingSequenceNumber
self.seq_no[pair] = msg['seq']
delta = {BID: [], ASK: []}
s = BID if msg['side'] == 'buy' else ASK
price = Decimal(msg['price'])
amount = Decimal(msg['qty'])
if amount == 0:
delta[s].append((price, 0))
del self._l2_book[pair].book[s][price]
else:
delta[s].append((price, amount))
self._l2_book[pair].book[s][price] = amount
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
delta=delta,
sequence_number=msg['seq'],
raw=msg)
async def _funding(self, msg: dict, pair: str, timestamp: float):
if 'funding_rate' in msg:
f = Funding(self.id,
pair,
None,
msg['funding_rate'],
self.timestamp_normalize(
msg['next_funding_rate_time']),
self.timestamp_normalize(msg['time']),
predicted_rate=msg['funding_rate_prediction'],
raw=msg)
await self.callback(FUNDING, f, timestamp)
oi = msg['openInterest']
if pair in self._open_interest_cache and oi == self._open_interest_cache[
pair]:
return
self._open_interest_cache[pair] = oi
o = OpenInterest(self.id,
pair,
oi,
self.timestamp_normalize(msg['time']),
raw=msg)
await self.callback(OPEN_INTEREST, o, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'event' in msg:
if msg['event'] == 'info':
return
elif msg['event'] == 'subscribed':
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
# As per Kraken support: websocket product_id is uppercase version of the REST API symbols
pair = self.exchange_symbol_to_std_symbol(
msg['product_id'].lower())
if msg['feed'] == 'trade':
await self._trade(msg, pair, timestamp)
elif msg['feed'] == 'trade_snapshot':
return
elif msg['feed'] == 'ticker_lite':
await self._ticker(msg, pair, timestamp)
elif msg['feed'] == 'ticker':
await self._funding(msg, pair, timestamp)
elif msg['feed'] == 'book_snapshot':
await self._book_snapshot(msg, pair, timestamp)
elif msg['feed'] == 'book':
await self._book(msg, pair, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
class Bybit(Feed):
id = BYBIT
websocket_channels = {
L2_BOOK: 'orderBook_200.100ms',
TRADES: 'trade',
FILLS: 'execution',
ORDER_INFO: 'order',
INDEX: 'instrument_info.100ms',
OPEN_INTEREST: 'instrument_info.100ms',
FUNDING: 'instrument_info.100ms',
CANDLES: 'klineV2',
LIQUIDATIONS: 'liquidation'
}
websocket_endpoints = [
WebsocketEndpoint('wss://stream.bybit.com/realtime',
channel_filter=(websocket_channels[L2_BOOK],
websocket_channels[TRADES],
websocket_channels[INDEX],
websocket_channels[OPEN_INTEREST],
websocket_channels[FUNDING],
websocket_channels[CANDLES],
websocket_channels[LIQUIDATIONS]),
instrument_filter=('QUOTE', ('USD', )),
sandbox='wss://stream-testnet.bybit.com/realtime',
options={'compression': None}),
WebsocketEndpoint(
'wss://stream.bybit.com/realtime_public',
channel_filter=(websocket_channels[L2_BOOK],
websocket_channels[TRADES],
websocket_channels[INDEX],
websocket_channels[OPEN_INTEREST],
websocket_channels[FUNDING],
websocket_channels[CANDLES],
websocket_channels[LIQUIDATIONS]),
instrument_filter=('QUOTE', ('USDT', )),
sandbox='wss://stream-testnet.bybit.com/realtime_public',
options={'compression': None}),
WebsocketEndpoint(
'wss://stream.bybit.com/realtime_private',
channel_filter=(websocket_channels[ORDER_INFO],
websocket_channels[FILLS]),
instrument_filter=('QUOTE', ('USDT', )),
sandbox='wss://stream-testnet.bybit.com/realtime_private',
options={'compression': None}),
]
rest_endpoints = [
RestEndpoint('https://api.bybit.com',
routes=Routes('/v2/public/symbols'))
]
valid_candle_intervals = {
'1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '1d', '1w',
'1M'
}
candle_interval_map = {
'1m': '1',
'3m': '3',
'5m': '5',
'15m': '15',
'30m': '30',
'1h': '60',
'2h': '120',
'4h': '240',
'6h': '360',
'1d': 'D',
'1w': 'W',
'1M': 'M'
}
@classmethod
def timestamp_normalize(cls, ts: Union[int, dt]) -> float:
if isinstance(ts, int):
return ts / 1000.0
else:
return ts.timestamp()
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
# PERPETUAL & FUTURES
for symbol in data['result']:
base = symbol['base_currency']
quote = symbol['quote_currency']
stype = PERPETUAL
expiry = None
if not symbol['name'].endswith(quote):
stype = FUTURES
year = symbol['name'].replace(base + quote, '')[-2:]
expiry = year + symbol['alias'].replace(base + quote, '')[-4:]
s = Symbol(base, quote, type=stype, expiry_date=expiry)
ret[s.normalized] = symbol['name']
info['tick_size'][
s.normalized] = symbol['price_filter']['tick_size']
info['instrument_type'][s.normalized] = stype
return ret, info
def __reset(self):
self._instrument_info_cache = {}
self._l2_book = {}
async def _candle(self, msg: dict, timestamp: float):
'''
{
"topic": "klineV2.1.BTCUSD", //topic name
"data": [{
"start": 1572425640, //start time of the candle
"end": 1572425700, //end time of the candle
"open": 9200, //open price
"close": 9202.5, //close price
"high": 9202.5, //max price
"low": 9196, //min price
"volume": 81790, //volume
"turnover": 8.889247899999999, //turnover
"confirm": False, //snapshot flag (indicates if candle is closed or not)
"cross_seq": 297503466,
"timestamp": 1572425676958323 //cross time
}],
"timestamp_e6": 1572425677047994 //server time
}
'''
symbol = self.exchange_symbol_to_std_symbol(
msg['topic'].split(".")[-1])
ts = msg['timestamp_e6'] / 1_000_000
for entry in msg['data']:
if self.candle_closed_only and not entry['confirm']:
continue
c = Candle(self.id,
symbol,
entry['start'],
entry['end'],
self.candle_interval,
entry['confirm'],
Decimal(entry['open']),
Decimal(entry['close']),
Decimal(entry['high']),
Decimal(entry['low']),
Decimal(entry['volume']),
None,
ts,
raw=entry)
await self.callback(CANDLES, c, timestamp)
async def _liquidation(self, msg: dict, timestamp: float):
'''
{
'topic': 'liquidation.EOSUSDT',
'data': {
'symbol': 'EOSUSDT',
'side': 'Buy',
'price': '3.950',
'qty': '58.0',
'time': 1632586154956
}
}
'''
liq = Liquidation(self.id,
self.exchange_symbol_to_std_symbol(
msg['data']['symbol']),
BUY if msg['data']['side'] == 'Buy' else SELL,
Decimal(msg['data']['qty']),
Decimal(msg['data']['price']),
None,
None,
self.timestamp_normalize(msg['data']['time']),
raw=msg)
await self.callback(LIQUIDATIONS, liq, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if "success" in msg:
if msg['success']:
if msg['request']['op'] == 'auth':
LOG.debug("%s: Authenticated successful", conn.uuid)
elif msg['request']['op'] == 'subscribe':
LOG.debug("%s: Subscribed to channels: %s", conn.uuid,
msg['request']['args'])
else:
LOG.warning("%s: Unhandled 'successs' message received",
conn.uuid)
else:
LOG.error("%s: Error from exchange %s", conn.uuid, msg)
elif msg["topic"].startswith('trade'):
await self._trade(msg, timestamp)
elif msg["topic"].startswith('orderBook'):
await self._book(msg, timestamp)
elif msg['topic'].startswith('liquidation'):
await self._liquidation(msg, timestamp)
elif "instrument_info" in msg["topic"]:
await self._instrument_info(msg, timestamp)
elif "order" in msg["topic"]:
await self._order(msg, timestamp)
elif "execution" in msg["topic"]:
await self._execution(msg, timestamp)
elif 'klineV2' in msg['topic'] or 'candle' in msg['topic']:
await self._candle(msg, timestamp)
# elif "position" in msg["topic"]:
# await self._balances(msg, timestamp)
else:
LOG.warning("%s: Unhandled message type %s", conn.uuid, msg)
async def subscribe(self, connection: AsyncConnection):
self.__reset()
for chan in connection.subscription:
if not self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
for pair in connection.subscription[chan]:
sym = str_to_symbol(
self.exchange_symbol_to_std_symbol(pair))
if self.exchange_channel_to_std(chan) == CANDLES:
c = chan if sym.quote == 'USD' else 'candle'
sub = [
f"{c}.{self.candle_interval_map[self.candle_interval]}.{pair}"
]
else:
sub = [f"{chan}.{pair}"]
await connection.write(
json.dumps({
"op": "subscribe",
"args": sub
}))
else:
await connection.write(
json.dumps({
"op": "subscribe",
"args": [f"{chan}"]
}))
async def _instrument_info(self, msg: dict, timestamp: float):
"""
### Snapshot type update
{
"topic": "instrument_info.100ms.BTCUSD",
"type": "snapshot",
"data": {
"id": 1,
"symbol": "BTCUSD", //instrument name
"last_price_e4": 81165000, //the latest price
"last_tick_direction": "ZeroPlusTick", //the direction of last tick:PlusTick,ZeroPlusTick,MinusTick,
//ZeroMinusTick
"prev_price_24h_e4": 81585000, //the price of prev 24h
"price_24h_pcnt_e6": -5148, //the current last price percentage change from prev 24h price
"high_price_24h_e4": 82900000, //the highest price of prev 24h
"low_price_24h_e4": 79655000, //the lowest price of prev 24h
"prev_price_1h_e4": 81395000, //the price of prev 1h
"price_1h_pcnt_e6": -2825, //the current last price percentage change from prev 1h price
"mark_price_e4": 81178500, //mark price
"index_price_e4": 81172800, //index price
"open_interest": 154418471, //open interest quantity - Attention, the update is not
//immediate - slowest update is 1 minute
"open_value_e8": 1997561103030, //open value quantity - Attention, the update is not
//immediate - the slowest update is 1 minute
"total_turnover_e8": 2029370141961401, //total turnover
"turnover_24h_e8": 9072939873591, //24h turnover
"total_volume": 175654418740, //total volume
"volume_24h": 735865248, //24h volume
"funding_rate_e6": 100, //funding rate
"predicted_funding_rate_e6": 100, //predicted funding rate
"cross_seq": 1053192577, //sequence
"created_at": "2018-11-14T16:33:26Z",
"updated_at": "2020-01-12T18:25:16Z",
"next_funding_time": "2020-01-13T00:00:00Z", //next funding time
//the rest time to settle funding fee
"countdown_hour": 6 //the remaining time to settle the funding fee
},
"cross_seq": 1053192634,
"timestamp_e6": 1578853524091081 //the timestamp when this information was produced
}
### Delta type update
{
"topic": "instrument_info.100ms.BTCUSD",
"type": "delta",
"data": {
"delete": [],
"update": [
{
"id": 1,
"symbol": "BTCUSD",
"prev_price_24h_e4": 81565000,
"price_24h_pcnt_e6": -4904,
"open_value_e8": 2000479681106,
"total_turnover_e8": 2029370495672976,
"turnover_24h_e8": 9066215468687,
"volume_24h": 735316391,
"cross_seq": 1053192657,
"created_at": "2018-11-14T16:33:26Z",
"updated_at": "2020-01-12T18:25:25Z"
}
],
"insert": []
},
"cross_seq": 1053192657,
"timestamp_e6": 1578853525691123
}
"""
update_type = msg['type']
if update_type == 'snapshot':
updates = [msg['data']]
else:
updates = msg['data']['update']
for info in updates:
if msg['topic'] in self._instrument_info_cache and self._instrument_info_cache[
msg['topic']] == updates:
continue
else:
self._instrument_info_cache[msg['topic']] = updates
ts = int(msg['timestamp_e6']) / 1_000_000
if 'open_interest' in info:
oi = OpenInterest(self.id,
self.exchange_symbol_to_std_symbol(
info['symbol']),
Decimal(info['open_interest']),
ts,
raw=info)
await self.callback(OPEN_INTEREST, oi, timestamp)
if 'index_price_e4' in info:
i = Index(self.id,
self.exchange_symbol_to_std_symbol(info['symbol']),
Decimal(info['index_price_e4']) * Decimal('1e-4'),
ts,
raw=info)
await self.callback(INDEX, i, timestamp)
if 'funding_rate_e6' in info:
f = Funding(
self.id,
self.exchange_symbol_to_std_symbol(info['symbol']),
None,
Decimal(info['funding_rate_e6']) * Decimal('1e-6'),
info['next_funding_time'].timestamp(),
ts,
predicted_rate=Decimal(info['predicted_funding_rate_e6']) *
Decimal('1e-6'),
raw=info)
await self.callback(FUNDING, f, timestamp)
async def _trade(self, msg: dict, timestamp: float):
"""
{"topic":"trade.BTCUSD",
"data":[
{
"timestamp":"2019-01-22T15:04:33.461Z",
"symbol":"BTCUSD",
"side":"Buy",
"size":980,
"price":3563.5,
"tick_direction":"PlusTick",
"trade_id":"9d229f26-09a8-42f8-aff3-0ba047b0449d",
"cross_seq":163261271}]}
"""
data = msg['data']
for trade in data:
if isinstance(trade['trade_time_ms'], str):
ts = int(trade['trade_time_ms'])
else:
ts = trade['trade_time_ms']
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(trade['symbol']),
BUY if trade['side'] == 'Buy' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
self.timestamp_normalize(ts),
id=trade['trade_id'],
raw=trade)
await self.callback(TRADES, t, timestamp)
async def _book(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['topic'].split('.')[-1])
update_type = msg['type']
data = msg['data']
delta = {BID: [], ASK: []}
if update_type == 'snapshot':
delta = None
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
# the USDT perpetual data is under the order_book key
if 'order_book' in data:
data = data['order_book']
for update in data:
side = BID if update['side'] == 'Buy' else ASK
self._l2_book[pair].book[side][Decimal(
update['price'])] = Decimal(update['size'])
else:
for delete in data['delete']:
side = BID if delete['side'] == 'Buy' else ASK
price = Decimal(delete['price'])
delta[side].append((price, 0))
del self._l2_book[pair].book[side][price]
for utype in ('update', 'insert'):
for update in data[utype]:
side = BID if update['side'] == 'Buy' else ASK
price = Decimal(update['price'])
amount = Decimal(update['size'])
delta[side].append((price, amount))
self._l2_book[pair].book[side][price] = amount
# timestamp is in microseconds
ts = msg['timestamp_e6']
if isinstance(ts, str):
ts = int(ts)
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=ts / 1000000,
raw=msg,
delta=delta)
async def _order(self, msg: dict, timestamp: float):
"""
{
"topic": "order",
"action": "",
"data": [
{
"order_id": "xxxxxxxx-xxxx-xxxx-9a8f-4a973eb5c418",
"order_link_id": "",
"symbol": "BTCUSDT",
"side": "Buy",
"order_type": "Limit",
"price": 11000,
"qty": 0.001,
"leaves_qty": 0.001,
"last_exec_price": 0,
"cum_exec_qty": 0,
"cum_exec_value": 0,
"cum_exec_fee": 0,
"time_in_force": "GoodTillCancel",
"create_type": "CreateByUser",
"cancel_type": "UNKNOWN",
"order_status": "New",
"take_profit": 0,
"stop_loss": 0,
"trailing_stop": 0,
"reduce_only": false,
"close_on_trigger": false,
"create_time": "2020-08-12T21:18:40.780039678Z",
"update_time": "2020-08-12T21:18:40.787986415Z"
}
]
}
"""
order_status = {
'Created': SUBMITTING,
'Rejected': FAILED,
'New': OPEN,
'PartiallyFilled': PARTIAL,
'Filled': FILLED,
'Cancelled': CANCELLED,
'PendingCancel': CANCELLING
}
for i in range(len(msg['data'])):
data = msg['data'][i]
oi = OrderInfo(
self.id,
self.exchange_symbol_to_std_symbol(data['symbol']),
data["order_id"],
BUY if data["side"] == 'Buy' else SELL,
order_status[data["order_status"]],
LIMIT if data['order_type'] == 'Limit' else MARKET,
Decimal(data['price']),
Decimal(data['cum_exec_qty']),
Decimal(data['qty']) - Decimal(data['cum_exec_qty']),
self.timestamp_normalize(
data.get('update_time') or data.get('O')
or data.get('timestamp')),
raw=data,
)
await self.callback(ORDER_INFO, oi, timestamp)
async def _execution(self, msg: dict, timestamp: float):
'''
{
"topic": "execution",
"data": [
{
"symbol": "BTCUSD",
"side": "Buy",
"order_id": "xxxxxxxx-xxxx-xxxx-9a8f-4a973eb5c418",
"exec_id": "xxxxxxxx-xxxx-xxxx-8b66-c3d2fcd352f6",
"order_link_id": "",
"price": "8300",
"order_qty": 1,
"exec_type": "Trade",
"exec_qty": 1,
"exec_fee": "0.00000009",
"leaves_qty": 0,
"is_maker": false,
"trade_time": "2020-01-14T14:07:23.629Z" // trade time
}
]
}
'''
for entry in msg['data']:
symbol = self.exchange_symbol_to_std_symbol(entry['symbol'])
f = Fill(self.id,
symbol,
BUY if entry['side'] == 'Buy' else SELL,
Decimal(entry['exec_qty']),
Decimal(entry['price']),
Decimal(entry['exec_fee']),
entry['exec_id'],
entry['order_id'],
None,
MAKER if entry['is_maker'] else TAKER,
entry['trade_time'].timestamp(),
raw=entry)
await self.callback(FILLS, f, timestamp)
# async def _balances(self, msg: dict, timestamp: float):
# for i in range(len(msg['data'])):
# data = msg['data'][i]
# symbol = self.exchange_symbol_to_std_symbol(data['symbol'])
# await self.callback(BALANCES, feed=self.id, symbol=symbol, data=data, receipt_timestamp=timestamp)
async def authenticate(self, conn: AsyncConnection):
if any(
self.is_authenticated_channel(
self.exchange_channel_to_std(chan))
for chan in conn.subscription):
auth = self._auth(self.key_id, self.key_secret)
LOG.debug(
f"{conn.uuid}: Sending authentication request with message {auth}"
)
await conn.write(auth)
def _auth(self, key_id: str, key_secret: str) -> str:
# https://bybit-exchange.github.io/docs/inverse/#t-websocketauthentication
expires = int((time.time() + 60)) * 1000
signature = str(
hmac.new(bytes(key_secret, 'utf-8'),
bytes(f'GET/realtime{expires}', 'utf-8'),
digestmod='sha256').hexdigest())
return json.dumps({'op': 'auth', 'args': [key_id, expires, signature]})
class dYdX(Feed, dYdXRestMixin):
id = DYDX
websocket_endpoints = [WebsocketEndpoint('wss://api.dydx.exchange/v3/ws')]
rest_endpoints = [
RestEndpoint('https://api.dydx.exchange', routes=Routes('/v3/markets'))
]
websocket_channels = {
L2_BOOK: 'v3_orderbook',
TRADES: 'v3_trades',
}
request_limit = 10
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for symbol, entry in data['markets'].items():
if entry['status'] != 'ONLINE':
continue
stype = entry['type'].lower()
s = Symbol(entry['baseAsset'], entry['quoteAsset'], type=stype)
ret[s.normalized] = symbol
info['tick_size'][s.normalized] = entry['tickSize']
info['instrument_type'][s.normalized] = stype
return ret, info
def __reset(self):
self._l2_book = {}
self._offsets = {}
async def _book(self, msg: dict, timestamp: float):
pair = self.exchange_symbol_to_std_symbol(msg['id'])
delta = {BID: [], ASK: []}
if msg['type'] == 'channel_data':
updated = False
offset = int(msg['contents']['offset'])
for side, key in ((BID, 'bids'), (ASK, 'asks')):
for data in msg['contents'][key]:
price = Decimal(data[0])
amount = Decimal(data[1])
if price in self._offsets[
pair] and offset < self._offsets[pair][price]:
continue
updated = True
self._offsets[pair][price] = offset
delta[side].append((price, amount))
if amount == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
else:
self._l2_book[pair].book[side][price] = amount
if updated:
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
delta=delta,
raw=msg)
else:
# snapshot
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._offsets[pair] = {}
for side, data in msg['contents'].items():
side = BID if side == 'bids' else ASK
for entry in data:
self._offsets[pair][Decimal(entry['price'])] = int(
entry['offset'])
size = Decimal(entry['size'])
if size > 0:
self._l2_book[pair].book[side][Decimal(
entry['price'])] = size
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
delta=None,
raw=msg)
async def _trade(self, msg: dict, timestamp: float):
"""
update:
{
'type': 'channel_data',
'connection_id': '7b4abf85-f9eb-4f6e-82c0-5479ad5681e9',
'message_id': 18,
'id': 'DOGE-USD',
'channel': 'v3_trades',
'contents': {
'trades': [{
'size': '390',
'side': 'SELL',
'price': '0.2334',
'createdAt': datetime.datetime(2021, 6, 23, 22, 36, 34, 520000, tzinfo=datetime.timezone.utc)
}]
}
}
initial message:
{
'type': 'subscribed',
'connection_id': 'ccd8b74c-97b3-491d-a9fc-4a92a171296e',
'message_id': 4,
'channel': 'v3_trades',
'id': 'UNI-USD',
'contents': {
'trades': [{
'side': 'BUY',
'size': '384.1',
'price': '17.23',
'createdAt': datetime.datetime(2021, 6, 23, 20, 28, 25, 465000, tzinfo=datetime.timezone.utc)
},
{
'side': 'SELL',
'size': '384.1',
'price': '17.138',
'createdAt': datetime.datetime(2021, 6, 23, 20, 22, 26, 466000, tzinfo=datetime.timezone.utc)},
}]
}
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['id'])
for trade in msg['contents']['trades']:
t = Trade(self.id,
pair,
BUY if trade['side'] == 'BUY' else SELL,
Decimal(trade['size']),
Decimal(trade['price']),
self.timestamp_normalize(trade['createdAt']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['type'] == 'channel_data' or msg['type'] == 'subscribed':
chan = self.exchange_channel_to_std(msg['channel'])
if chan == L2_BOOK:
await self._book(msg, timestamp)
elif chan == TRADES:
await self._trade(msg, timestamp)
else:
LOG.warning("%s: unexpected channel type received: %s",
self.id, msg)
elif msg['type'] == 'connected':
return
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan, symbols in self.subscription.items():
for symbol in symbols:
msg = {"type": "subscribe", "channel": chan, "id": symbol}
if self.exchange_channel_to_std(chan) == L2_BOOK:
msg['includeOffsets'] = True
await conn.write(json.dumps(msg))
class HuobiDM(Feed):
id = HUOBI_DM
websocket_endpoints = [WebsocketEndpoint('wss://www.hbdm.com/ws')]
rest_endpoints = [
RestEndpoint('https://www.hbdm.com',
routes=Routes('/api/v1/contract_contract_info'))
]
websocket_channels = {
L2_BOOK: 'depth.step0',
TRADES: 'trade.detail',
}
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for e in data['data']:
# Pricing is all in USD, see https://huobiglobal.zendesk.com/hc/en-us/articles/360000113102-Introduction-of-Huobi-Futures
s = Symbol(e['symbol'],
'USD',
type=FUTURES,
expiry_date=e['contract_code'].replace(e['symbol'], ''))
ret[s.normalized] = e['contract_code']
info['tick_size'][s.normalized] = e['price_tick']
info['instrument_type'][s.normalized] = FUTURES
return ret, info
def __reset(self):
self._l2_book = {}
async def _book(self, msg: dict, timestamp: float):
"""
{
'ch':'market.BTC_CW.depth.step0',
'ts':1565857755564,
'tick':{
'mrid':14848858327,
'id':1565857755,
'bids':[
[ Decimal('9829.99'), 1], ...
]
'asks':[
[ 9830, 625], ...
]
},
'ts':1565857755552,
'version':1565857755,
'ch':'market.BTC_CW.depth.step0'
}
"""
pair = self.exchange_symbol_to_std_symbol(msg['ch'].split('.')[1])
data = msg['tick']
# When Huobi Delists pairs, empty updates still sent:
# {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951241196, 'tick': {'mrid': 50651100044, 'id': 1606951241, 'ts': 1606951241195, 'version': 1606951241, 'ch': 'market.AKRO-USD.depth.step0'}}
# {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951242297, 'tick': {'mrid': 50651100044, 'id': 1606951242, 'ts': 1606951242295, 'version': 1606951242, 'ch': 'market.AKRO-USD.depth.step0'}}
if 'bids' in data and 'asks' in data:
if pair not in self._l2_book:
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
self._l2_book[pair].book.bids = {
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}
self._l2_book[pair].book.asks = {
Decimal(price): Decimal(amount)
for price, amount in data['asks']
}
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(
msg['ts']),
raw=msg)
async def _trade(self, msg: dict, timestamp: float):
"""
{
'ch': 'market.btcusd.trade.detail',
'ts': 1549773923965,
'tick': {
'id': 100065340982,
'ts': 1549757127140,
'data': [{'id': '10006534098224147003732', 'amount': Decimal('0.0777'), 'price': Decimal('3669.69'), 'direction': 'buy', 'ts': 1549757127140}]}
}
"""
for trade in msg['tick']['data']:
t = Trade(self.id,
self.exchange_symbol_to_std_symbol(
msg['ch'].split('.')[1]),
BUY if trade['direction'] == 'buy' else SELL,
Decimal(trade['amount']),
Decimal(trade['price']),
self.timestamp_normalize(trade['ts']),
id=str(trade['id']),
raw=trade)
await self.callback(TRADES, t, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
# unzip message
msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS)
msg = json.loads(msg, parse_float=Decimal)
# Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it
if 'ping' in msg:
await conn.write(json.dumps({'pong': msg['ping']}))
elif 'status' in msg and msg['status'] == 'ok':
return
elif 'ch' in msg:
if 'trade' in msg['ch']:
await self._trade(msg, timestamp)
elif 'depth' in msg['ch']:
await self._book(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
client_id = 0
for chan, symbols in conn.subscription.items():
for symbol in symbols:
client_id += 1
await conn.write(
json.dumps({
"sub": f"market.{symbol}.{chan}",
"id": str(client_id)
}))
class Bitstamp(Feed, BitstampRestMixin):
id = BITSTAMP
# API documentation: https://www.bitstamp.net/websocket/v2/
websocket_endpoints = [WebsocketEndpoint('wss://ws.bitstamp.net/', options={'compression': None})]
rest_endpoints = [RestEndpoint('https://www.bitstamp.net', routes=Routes('/api/v2/trading-pairs-info/', l2book='/api/v2/order_book/{}'))]
websocket_channels = {
L3_BOOK: 'detail_order_book',
L2_BOOK: 'diff_order_book',
TRADES: 'live_trades',
}
request_limit = 13
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1_000_000.0
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
ret = {}
info = {'instrument_type': {}}
for d in data:
if d['trading'] != 'Enabled':
continue
base, quote = d['name'].split("/")
s = Symbol(base, quote)
symbol = d['url_symbol']
ret[s.normalized] = symbol
info['instrument_type'][s.normalized] = s.type
return ret, info
async def _process_l2_book(self, msg: dict, timestamp: float):
data = msg['data']
chan = msg['channel']
ts = int(data['microtimestamp'])
pair = self.exchange_symbol_to_std_symbol(chan.split('_')[-1])
delta = {BID: [], ASK: []}
if pair in self.last_update_id:
if data['timestamp'] < self.last_update_id[pair]:
return
else:
del self.last_update_id[pair]
for side in (BID, ASK):
for update in data[side + 's']:
price = Decimal(update[0])
size = Decimal(update[1])
if size == 0:
if price in self._l2_book[pair].book[side]:
del self._l2_book[pair].book[side][price]
delta[side].append((price, size))
else:
self._l2_book[pair].book[side][price] = size
delta[side].append((price, size))
await self.book_callback(L2_BOOK, self._l2_book[pair], timestamp, timestamp=self.timestamp_normalize(ts), delta=delta, raw=msg)
async def _process_l3_book(self, msg: dict, timestamp: float):
data = msg['data']
chan = msg['channel']
ts = int(data['microtimestamp'])
pair = self.exchange_symbol_to_std_symbol(chan.split('_')[-1])
book = OrderBook(self.id, pair, max_depth=self.max_depth)
for side in (BID, ASK):
for price, size, order_id in data[side + 's']:
price = Decimal(price)
size = Decimal(size)
if price in book.book[side]:
book.book[side][price][order_id] = size
else:
book.book[side][price] = {order_id: size}
self._l3_book[pair] = book
await self.book_callback(L3_BOOK, self._l3_book[pair], timestamp, timestamp=self.timestamp_normalize(ts), raw=msg)
async def _trades(self, msg: dict, timestamp: float):
"""
{'data':
{
'microtimestamp': '1562650233964229', // Event time (micros)
'amount': Decimal('0.014140160000000001'), // Quantity
'buy_order_id': 3709484695, // Buyer order ID
'sell_order_id': 3709484799, // Seller order ID
'amount_str': '0.01414016', // Quantity string
'price_str': '12700.00', // Price string
'timestamp': '1562650233', // Event time
'price': Decimal('12700.0'), // Price
'type': 1,
'id': 93215787
},
'event': 'trade',
'channel': 'live_trades_btcusd'
}
"""
data = msg['data']
chan = msg['channel']
pair = self.exchange_symbol_to_std_symbol(chan.split('_')[-1])
t = Trade(
self.id,
pair,
BUY if data['type'] == 0 else SELL,
Decimal(data['amount']),
Decimal(data['price']),
self.timestamp_normalize(int(data['microtimestamp'])),
id=str(data['id']),
raw=msg
)
await self.callback(TRADES, t, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'bts' in msg['event']:
if msg['event'] == 'bts:connection_established':
pass
elif msg['event'] == 'bts:subscription_succeeded':
pass
else:
LOG.warning("%s: Unexpected message %s", self.id, msg)
elif msg['event'] == 'trade':
await self._trades(msg, timestamp)
elif msg['event'] == 'data':
if msg['channel'].startswith('diff_order_book'):
await self._process_l2_book(msg, timestamp)
if msg['channel'].startswith('detail_order_book'):
await self._process_l3_book(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _snapshot(self, pairs: list, conn: AsyncConnection):
await asyncio.sleep(5)
urls = [self.rest_endpoints[0].route('l2book', self.sandbox).format(sym) for sym in pairs]
results = [await self.http_conn.read(url) for url in urls]
results = [json.loads(resp, parse_float=Decimal) for resp in results]
for r, pair in zip(results, pairs):
std_pair = self.exchange_symbol_to_std_symbol(pair) if pair else 'BTC-USD'
self.last_update_id[std_pair] = r['timestamp']
self._l2_book[std_pair] = OrderBook(self.id, std_pair, max_depth=self.max_depth, asks={Decimal(u[0]): Decimal(u[1]) for u in r['asks']}, bids={Decimal(u[0]): Decimal(u[1]) for u in r['bids']})
await self.book_callback(L2_BOOK, self._l2_book[std_pair], time.time(), timestamp=float(r['timestamp']), raw=r)
async def subscribe(self, conn: AsyncConnection):
snaps = []
self.last_update_id = {}
for chan in self.subscription:
for pair in self.subscription[chan]:
await conn.write(
json.dumps({
"event": "bts:subscribe",
"data": {
"channel": f"{chan}_{pair}"
}
}))
if 'diff_order_book' in chan:
snaps.append(pair)
await self._snapshot(snaps, conn)
class BitDotCom(Feed):
id = BITDOTCOM
websocket_endpoints = [
WebsocketEndpoint('wss://spot-ws.bit.com',
instrument_filter=('TYPE', (SPOT, )),
sandbox='wss://betaspot-ws.bitexch.dev'),
WebsocketEndpoint('wss://ws.bit.com',
instrument_filter=('TYPE', (FUTURES, OPTION,
PERPETUAL)),
sandbox='wss://betaws.bitexch.dev'),
]
rest_endpoints = [
RestEndpoint('https://spot-api.bit.com',
instrument_filter=('TYPE', (SPOT, )),
sandbox='https://betaspot-api.bitexch.dev',
routes=Routes('/spot/v1/instruments',
authentication='/spot/v1/ws/auth')),
RestEndpoint('https://api.bit.com',
instrument_filter=('TYPE', (OPTION, FUTURES, PERPETUAL)),
sandbox='https://betaapi.bitexch.dev',
routes=Routes('/v1/instruments?currency={}&active=true',
currencies='/v1/currencies',
authentication='/v1/ws/auth'))
]
websocket_channels = {
L2_BOOK: 'depth',
TRADES: 'trade',
TICKER: 'ticker',
ORDER_INFO: 'order',
BALANCES: 'account',
FILLS: 'user_trade',
# funding rates paid and received
}
request_limit = 10
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._sequence_no = defaultdict(int)
@classmethod
def _symbol_endpoint_prepare(cls,
ep: RestEndpoint) -> Union[List[str], str]:
if ep.routes.currencies:
ret = cls.http_sync.read(ep.route('currencies'),
json=True,
uuid=cls.id)
return [
ep.route('instruments').format(currency)
for currency in ret['data']['currencies']
]
return ep.route('instruments')
@classmethod
def timestamp_normalize(cls, ts: float) -> float:
return ts / 1000.0
@classmethod
def _parse_symbol_data(cls, data: list) -> Tuple[Dict, Dict]:
ret = {}
info = defaultdict(dict)
for entry in data:
if entry['code'] != 0:
raise ValueError('%s - Failed to collect instrument data - %s',
cls.id, entry['message'])
for mapping in entry['data']:
if 'category' in mapping:
expiry = None
strike = None
otype = None
if mapping['category'] == 'option':
stype = OPTION
strike = int(float(mapping['strike_price']))
expiry = cls.timestamp_normalize(
mapping['expiration_at'])
otype = mapping['option_type']
elif mapping['category'] == 'future':
if 'PERPETUAL' in mapping['instrument_id']:
stype = PERPETUAL
else:
stype = FUTURES
expiry = cls.timestamp_normalize(
mapping['expiration_at'])
s = Symbol(mapping['base_currency'],
mapping['quote_currency'],
type=stype,
option_type=otype,
expiry_date=expiry,
strike_price=strike)
ret[s.normalized] = mapping['instrument_id']
info['instrument_type'][s.normalized] = stype
else:
# Spot
s = Symbol(mapping['base_currency'],
mapping['quote_currency'],
type=SPOT)
ret[s.normalized] = mapping['pair']
info['instrument_type'][s.normalized] = SPOT
return ret, info
def __reset(self, conn: AsyncConnection):
if self.std_channel_to_exchange(L2_BOOK) in conn.subscription:
for pair in conn.subscription[self.std_channel_to_exchange(
L2_BOOK)]:
std_pair = self.exchange_symbol_to_std_symbol(pair)
if std_pair in self._l2_book:
del self._l2_book[std_pair]
if std_pair in self._sequence_no:
del self._sequence_no[std_pair]
def encode_list(self, item_list: list):
list_val = []
for item in item_list:
obj_val = self.encode_object(item)
list_val.append(obj_val)
output = '&'.join(list_val)
return '[' + output + ']'
def get_signature(self, api_path: str, param_map: dict):
str_to_sign = api_path + '&' + self.encode_object(param_map)
return hmac.new(self.key_secret.encode('utf-8'),
str_to_sign.encode('utf-8'),
digestmod=hashlib.sha256).hexdigest()
def encode_object(self, param_map: dict):
sorted_keys = sorted(param_map.keys())
ret_list = []
for key in sorted_keys:
val = param_map[key]
if isinstance(val, list):
list_val = self.encode_list(val)
ret_list.append(f'{key}={list_val}')
elif isinstance(val, dict):
dict_val = self.encode_object(val)
ret_list.append(f'{key}={dict_val}')
elif isinstance(val, bool):
bool_val = str(val).lower()
ret_list.append(f'{key}={bool_val}')
else:
general_val = str(val)
ret_list.append(f'{key}={general_val}')
sorted_list = sorted(ret_list)
return '&'.join(sorted_list)
async def authenticate(self, connection: AsyncConnection):
if not self.key_id or not self.key_secret:
return
if any([
self.is_authenticated_channel(self.exchange_channel_to_std(c))
for c in connection.subscription
]):
symbols = list(
set(itertools.chain(*connection.subscription.values())))
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(symbols[0]))
for ep in self.rest_endpoints:
if sym.type in ep.instrument_filter[1]:
ts = int(round(time.time() * 1000))
signature = self.get_signature(ep.routes.authentication,
{'timestamp': ts})
params = {'timestamp': ts, 'signature': signature}
ret = self.http_sync.read(
ep.route('authentication', sandbox=self.sandbox),
params=params,
headers={'X-Bit-Access-Key': self.key_id},
json=True)
if ret['code'] != 0 or 'token' not in ret['data']:
LOG.warning('%s: authentication failed: %s', ret)
token = ret['data']['token']
self._auth_token = token
return
async def subscribe(self, connection: AsyncConnection):
self.__reset(connection)
for chan, symbols in connection.subscription.items():
if len(symbols) == 0:
continue
stype = str_to_symbol(
self.exchange_symbol_to_std_symbol(symbols[0])).type
msg = {
'type': 'subscribe',
'channels': [chan],
'instruments' if stype in {PERPETUAL, FUTURES, OPTION} else 'pairs':
symbols,
}
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
msg['token'] = self._auth_token
await connection.write(json.dumps(msg))
async def _trade(self, data: dict, timestamp: float):
"""
{
'channel': 'trade',
'timestamp': 1639080717242,
'data': [{
'trade_id': '7016884324',
'instrument_id': 'BTC-PERPETUAL',
'price': '47482.50000000',
'qty': '6000.00000000',
'side': 'sell',
'sigma': '0.00000000',
'is_block_trade': False,
'created_at': 1639080717195
}]
}
"""
for t in data['data']:
trade = Trade(self.id,
self.exchange_symbol_to_std_symbol(
t.get('instrument_id') or t.get('pair')),
SELL if t['side'] == 'sell' else BUY,
Decimal(t['qty']),
Decimal(t['price']),
self.timestamp_normalize(t['created_at']),
id=t['trade_id'],
raw=t)
await self.callback(TRADES, trade, timestamp)
async def _book(self, data: dict, timestamp: float):
'''
Snapshot
{
'channel': 'depth',
'timestamp': 1639083660346,
'data': {
'type': 'snapshot',
'instrument_id': 'BTC-PERPETUAL',
'sequence': 1639042602148589825,
'bids': [
['47763.00000000', '20000.00000000'],
['47762.50000000', '6260.00000000'],
...
]
'asks': [
['47768.00000000', '10000.00000000'],
['47776.50000000', '20000.00000000'],
...
]
}
}
Delta
{
'channel': 'depth',
'timestamp': 1639083660401,
'data': {
'type': 'update',
'instrument_id': 'BTC-PERPETUAL',
'sequence': 1639042602148589842,
'prev_sequence': 1639042602148589841,
'changes': [
['sell', '47874.00000000', '0.00000000']
]
}
}
'''
if data['data']['type'] == 'update':
pair = self.exchange_symbol_to_std_symbol(
data['data'].get('instrument_id') or data['data'].get('pair'))
if data['data']['sequence'] != self._sequence_no[pair] + 1:
raise MissingSequenceNumber(
"Missing sequence number, restarting")
self._sequence_no[pair] = data['data']['sequence']
delta = {BID: [], ASK: []}
for side, price, amount in data['data']['changes']:
side = ASK if side == 'sell' else BID
price = Decimal(price)
amount = Decimal(amount)
if amount == 0:
delta[side].append((price, 0))
del self._l2_book[pair].book[side][price]
else:
delta[side].append((price, amount))
self._l2_book[pair].book[side][price] = amount
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(
data['timestamp']),
raw=data,
sequence_number=self._sequence_no[pair],
delta=delta)
else:
pair = self.exchange_symbol_to_std_symbol(
data['data'].get('instrument_id') or data['data'].get('pair'))
self._l2_book[pair] = OrderBook(
self.id,
pair,
max_depth=self.max_depth,
bids={
Decimal(price): Decimal(size)
for price, size in data['data']['bids']
},
asks={
Decimal(price): Decimal(size)
for price, size in data['data']['asks']
})
self._sequence_no[pair] = data['data']['sequence']
await self.book_callback(L2_BOOK,
self._l2_book[pair],
timestamp,
timestamp=self.timestamp_normalize(
data['timestamp']),
raw=data,
sequence_number=data['data']['sequence'])
async def _ticker(self, data: dict, timestamp: float):
'''
{
'channel': 'ticker',
'timestamp': 1639093870710,
'data': {
'time': 1639093870710,
'instrument_id': 'ETH-PERPETUAL',
'best_bid': '4155.85000000',
'best_ask': '4155.90000000',
'best_bid_qty': '2000.00000000',
'best_ask_qty': '3000.00000000',
'ask_sigma': '',
'bid_sigma': '',
'last_price': '4157.80000000',
'last_qty': '1000.00000000',
'open24h': '4436.75000000',
'high24h': '4490.00000000',
'low24h': '4086.60000000',
'price_change24h': '-0.06287260',
'volume24h': '1000218.00000000',
'open_interest': '7564685.00000000',
'funding_rate': '0.00025108',
'funding_rate8h': '0.00006396',
'mark_price': '4155.62874869',
'min_sell': '4030.50000000',
'max_buy': '4280.50000000'
}
}
'''
if data['data']['best_bid'] and data['data']['best_ask']:
t = Ticker(self.id,
self.exchange_symbol_to_std_symbol(
data['data'].get('instrument_id')
or data['data'].get('pair')),
Decimal(data['data']['best_bid']),
Decimal(data['data']['best_ask']),
self.timestamp_normalize(data['timestamp']),
raw=data)
await self.callback(TICKER, t, timestamp)
def _order_type_translate(self, t: str) -> str:
if t == 'limit':
return LIMIT
if t == 'market':
return MARKET
if t == 'stop-limit':
return STOP_LIMIT
if t == 'stop-market':
return STOP_MARKET
if t == 'trigger-limit':
return TRIGGER_LIMIT
if t == 'trigger-market':
return TRIGGER_MARKET
raise ValueError('Invalid order type detected %s', t)
def _status_translate(self, s: str) -> str:
if s == 'open':
return OPEN
if s == 'pending':
return PENDING
if s == 'filled':
return FILLED
if s == 'cancelled':
return CANCELLED
raise ValueError('Invalid order status detected %s', s)
async def _order(self, msg: dict, timestamp: float):
"""
{
"channel":"order",
"timestamp":1587994934089,
"data":[
{
"order_id":"1590",
"instrument_id":"BTC-1MAY20-8750-P",
"qty":"0.50000000",
"filled_qty":"0.10000000",
"remain_qty":"0.40000000",
"price":"0.16000000",
"avg_price":"0.16000000",
"side":"buy",
"order_type":"limit",
"time_in_force":"gtc",
"created_at":1587870609000,
"updated_at":1587870609000,
"status":"open",
"fee":"0.00002000",
"cash_flow":"-0.01600000",
"pnl":"0.00000000",
"is_liquidation": false,
"auto_price":"0.00000000",
"auto_price_type":"",
"taker_fee_rate": "0.00050000",
"maker_fee_rate": "0.00020000",
"label": "hedge",
"stop_price": "0.00000000",
"reduce_only": false,
"post_only": false,
"reject_post_only": false,
"mmp": false,
"reorder_index": 1
}
]
}
"""
for entry in msg['data']:
oi = OrderInfo(self.id,
self.exchange_symbol_to_std_symbol(
entry['instrument_id']),
entry['order_id'],
BUY if entry['side'] == 'buy' else SELL,
self._status_translate(entry['status']),
self._order_type_translate(entry['order_type']),
Decimal(entry['price']),
Decimal(entry['filled_qty']),
Decimal(entry['remain_qty']),
self.timestamp_normalize(entry['updated_at']),
raw=entry)
await self.callback(ORDER_INFO, oi, timestamp)
async def _balances(self, msg: dict, timestamp: float):
'''
Futures/Options
{
"channel":"account",
"timestamp":1589031930115,
"data":{
"user_id":"53345",
"currency":"BTC",
"cash_balance":"9999.94981346",
"available_balance":"9999.90496213",
"margin_balance":"9999.94981421",
"initial_margin":"0.04485208",
"maintenance_margin":"0.00000114",
"equity":"10000.00074364",
"pnl":"0.00746583",
"total_delta":"0.06207078",
"account_id":"8",
"mode":"regular",
"session_upl":"0.00081244",
"session_rpl":"0.00000021",
"option_value":"0.05092943",
"option_pnl":"0.00737943",
"option_session_rpl":"0.00000000",
"option_session_upl":"0.00081190",
"option_delta":"0.11279249",
"option_gamma":"0.00002905",
"option_vega":"4.30272923",
"option_theta":"-3.08908220",
"future_pnl":"0.00008640",
"future_session_rpl":"0.00000021",
"future_session_upl":"0.00000054",
"future_session_funding":"0.00000021",
"future_delta":"0.00955630",
"created_at":1588997840512,
"projected_info": {
"projected_initial_margin": "0.97919888",
"projected_maintenance_margin": "0.78335911",
"projected_total_delta": "3.89635553"
}
}
}
Spot
{
'channel': 'account',
'timestamp': 1641516119102,
'data': {
'user_id': '979394',
'balances': [
{
'currency': 'BTC',
'available': '31.02527500',
'frozen': '0.00000000'
},
{
'currency': 'ETH',
'available': '110.00000000',
'frozen': '0.00000000'
}
]
}
}
'''
if 'balances' in msg['data']:
# Spot
for balance in msg['data']['balances']:
b = Balance(self.id,
balance['currency'],
Decimal(balance['available']),
Decimal(balance['frozen']),
raw=msg)
await self.callback(BALANCES, b, timestamp)
else:
b = Balance(self.id,
msg['data']['currency'],
Decimal(msg['data']['cash_balance']),
Decimal(msg['data']['cash_balance']) -
Decimal(msg['data']['available_balance']),
raw=msg)
await self.callback(BALANCES, b, timestamp)
async def _fill(self, msg: dict, timestamp: float):
'''
{
"channel":"user_trade",
"timestamp":1588997059737,
"data":[
{
"trade_id":2388418,
"order_id":1384232,
"instrument_id":"BTC-26JUN20-6000-P",
"qty":"0.10000000",
"price":"0.01800000",
"sigma":"1.15054346",
"underlying_price":"9905.54000000",
"index_price":"9850.47000000",
"usd_price":"177.30846000",
"fee":"0.00005000",
"fee_rate":"0.00050000",
"side":"buy",
"created_at":1588997060000,
"is_taker":true,
"order_type":"limit",
"is_block_trade":false,
"label": "hedge"
}
]
}
'''
for entry in msg['data']:
f = Fill(self.id,
self.exchange_symbol_to_std_symbol(
entry['instrument_id']),
BUY if entry['side'] == 'buy' else SELL,
Decimal(entry['qty']),
Decimal(entry['price']),
Decimal(entry['fee']),
str(entry['trade_id']),
str(entry['order_id']),
self._order_type_translate(entry['order_type']),
None,
self.timestamp_normalize(entry['created_at']),
raw=entry)
await self.callback(FILLS, f, timestamp)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if msg['channel'] == 'depth':
await self._book(msg, timestamp)
elif msg['channel'] == 'trade':
await self._trade(msg, timestamp)
elif msg['channel'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['channel'] == 'order':
await self._order(msg, timestamp)
elif msg['channel'] == 'account':
await self._balances(msg, timestamp)
elif msg['channel'] == 'user_trade':
await self._fill(msg, timestamp)
elif msg['channel'] == 'subscription':
"""
{
'channel': 'subscription',
'timestamp': 1639080572093,
'data': {'code': 0, 'subscription': ['trade']}
}
"""
if msg['data']['code'] == 0:
return
else:
LOG.warning(
"%s: error received from exchange while subscribing: %s",
self.id, msg)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
class Blockchain(Feed):
id = BLOCKCHAIN
websocket_endpoints = [
WebsocketEndpoint(
'wss://ws.blockchain.info/mercury-gateway/v1/ws',
options={'origin': 'https://exchange.blockchain.com'})
]
rest_endpoints = [
RestEndpoint('https://api.blockchain.com',
routes=Routes('/mercury-gateway/v1/instruments'))
]
websocket_channels = {
L3_BOOK: 'l3',
L2_BOOK: 'l2',
TRADES: 'trades',
}
@classmethod
def _parse_symbol_data(cls, data: dict) -> Tuple[Dict, Dict]:
info = {'instrument_type': {}}
ret = {}
for entry in data:
if entry['status'] != 'open':
continue
base, quote = entry['symbol'].split("-")
s = Symbol(base, quote)
ret[s.normalized] = entry['symbol']
info['instrument_type'][s.normalized] = s.type
return ret, info
def __reset(self):
self.seq_no = None
self._l2_book = {}
self._l3_book = {}
async def _pair_l2_update(self, msg: str, timestamp: float):
delta = {BID: [], ASK: []}
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
if msg['event'] == 'snapshot':
# Reset the book
self._l2_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
for side in (BID, ASK):
for update in msg[side + 's']:
price = update['px']
qty = update['qty']
self._l2_book[pair].book[side][price] = qty
if qty <= 0:
del self._l2_book[pair].book[side][price]
delta[side].append((price, qty))
await self.book_callback(
L2_BOOK,
self._l2_book[pair],
timestamp,
raw=msg,
delta=delta if msg['event'] != 'snapshot' else None,
sequence_number=msg['seqnum'])
async def _handle_l2_msg(self, msg: str, timestamp: float):
"""
Subscribed message
{
"seqnum": 1,
"event": "subscribed",
"channel": "l2",
"symbol": "BTC-USD"
}
"""
if msg['event'] == 'subscribed':
LOG.debug("%s: Subscribed to L2 data for %s", self.id,
msg['symbol'])
elif msg['event'] in ['snapshot', 'updated']:
await self._pair_l2_update(msg, timestamp)
else:
LOG.warning("%s: Unexpected message %s", self.id, msg)
async def _pair_l3_update(self, msg: str, timestamp: float):
delta = {BID: [], ASK: []}
pair = self.exchange_symbol_to_std_symbol(msg['symbol'])
if msg['event'] == 'snapshot':
# Reset the book
self._l3_book[pair] = OrderBook(self.id,
pair,
max_depth=self.max_depth)
for side in (BID, ASK):
for update in msg[side + 's']:
price = update['px']
qty = update['qty']
order_id = update['id']
if qty <= 0:
del self._l3_book[pair].book[side][price][order_id]
else:
if price in self._l3_book[pair].book[side]:
self._l3_book[pair].book[side][price][order_id] = qty
else:
self._l3_book[pair].book[side][price] = {order_id: qty}
if len(self._l3_book[pair].book[side][price]) == 0:
del self._l3_book[pair].book[side][price]
delta[side].append((order_id, price, qty))
await self.book_callback(
L3_BOOK,
self._l3_book[pair],
timestamp,
raw=msg,
delta=delta if msg['event'] != 'snapshot' else None,
sequence_number=msg['seqnum'])
async def _handle_l3_msg(self, msg: str, timestamp: float):
if msg['event'] == 'subscribed':
LOG.debug("%s: Subscribed to L3 data for %s", self.id,
msg['symbol'])
elif msg['event'] in ['snapshot', 'updated']:
await self._pair_l3_update(msg, timestamp)
else:
LOG.warning("%s: Unexpected message %s", self.id, msg)
async def _trade(self, msg: dict, timestamp: float):
"""
trade msg example
{
"seqnum": 21,
"event": "updated",
"channel": "trades",
"symbol": "BTC-USD",
"timestamp": "2019-08-13T11:30:06.100140Z",
"side": "sell",
"qty": 8.5E-5,
"price": 11252.4,
"trade_id": "12884909920"
}
"""
t = Trade(
self.id,
msg['symbol'],
BUY if msg['side'] == 'buy' else SELL,
msg['qty'],
msg['price'],
self.timestamp_normalize(msg['timestamp']),
id=msg['trade_id'],
)
await self.callback(TRADES, t, timestamp)
async def _handle_trade_msg(self, msg: str, timestamp: float):
if msg['event'] == 'subscribed':
LOG.debug("%s: Subscribed to trades channel for %s", self.id,
msg['symbol'])
elif msg['event'] == 'updated':
await self._trade(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if self.seq_no is not None and msg['seqnum'] != self.seq_no + 1:
LOG.warning("%s: Missing sequence number detected!", self.id)
raise MissingSequenceNumber("Missing sequence number, restarting")
self.seq_no = msg['seqnum']
if 'channel' in msg:
if msg['channel'] == 'l2':
await self._handle_l2_msg(msg, timestamp)
elif msg['channel'] == 'l3':
await self._handle_l3_msg(msg, timestamp)
elif msg['channel'] == 'trades':
await self._handle_trade_msg(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
for pair in self.subscription[chan]:
await conn.write(
json.dumps({
"action": "subscribe",
"symbol": pair,
"channel": chan
}))