def add_feed(self, feed, **kwargs):
"""
feed: str or class
the feed (exchange) to add to the handler
kwargs: dict
if a string is used for the feed, kwargs will be passed to the
newly instantiated object
"""
if isinstance(feed, str):
if feed in EXCHANGE_MAP:
self.feeds.append((EXCHANGE_MAP[feed](config=self.config, **kwargs)))
else:
raise ValueError("Invalid feed specified")
else:
self.feeds.append((feed))
if self.raw_data_collection:
self.raw_data_collection.write_header(self.feeds[-1].id, json.dumps(self.feeds[-1]._feed_config))
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'id' in msg and msg['id'] == 100:
if not msg['error']:
LOG.info("%s: Auth request result: %s", conn.uuid,
msg['result']['status'])
msg = json.dumps({
"id":
101,
"method":
self.std_channel_to_exchange(BALANCES),
"params": []
})
LOG.debug(
f"{conn.uuid}: Subscribing to authenticated channels: {msg}"
)
await conn.write(msg)
else:
LOG.warning("%s: Auth unsuccessful: %s", conn.uuid, msg)
elif 'id' in msg and msg['id'] == 101:
if not msg['error']:
LOG.info("%s: Subscribe to auth channels request result: %s",
conn.uuid, msg['result']['status'])
else:
LOG.warning(f"{conn.uuid}: Subscription unsuccessful: {msg}")
elif 'id' in msg and msg['id'] == 1 and not msg['error']:
pass
elif 'accounts' in msg:
await self._user_data(msg, timestamp)
elif 'book' in msg:
await self._book(msg, timestamp)
elif 'trades' in msg:
await self._trade(msg, timestamp)
elif 'kline' in msg:
await self._candle(msg, timestamp)
elif 'result' in msg:
if 'error' in msg and msg['error'] is not None:
LOG.warning("%s: Error from exchange %s", conn.uuid, msg)
return
else:
LOG.warning("%s: Unhandled 'result' message: %s", conn.uuid,
msg)
else:
LOG.warning("%s: Invalid message type %s", conn.uuid, msg)
async def subscribe(self, connection: AsyncConnection, quote: str = None):
self.__reset()
for chan in self.channels if self.channels else self.subscription:
for pair in self.symbols if self.symbols else self.subscription[
chan]:
# Bybit uses separate addresses for difference quote currencies
if pair[-4:] == 'USDT' and quote != 'USDT':
continue
if pair[-3:] == 'USD' and quote != 'USD':
continue
await connection.send(
json.dumps({
"op": "subscribe",
"args": [f"{chan}.{pair}"]
}))
async def subscribe(self, conn: AsyncConnection):
self.__reset()
symbol_channels = list(self.get_channel_symbol_combinations())
LOG.info("%s: Got %r combinations of pairs and channels", self.id,
len(symbol_channels))
if len(symbol_channels) == 0:
LOG.info("%s: No websocket subscription", self.id)
return False
# Avoid error "Max frame length of 65536 has been exceeded" by limiting requests to some args
for chunk in split.list_by_max_items(symbol_channels, 33):
LOG.info("%s: Subscribe to %s args from %r to %r", self.id,
len(chunk), chunk[0], chunk[-1])
request = {"op": "subscribe", "args": chunk}
await conn.send(json.dumps(request))
async def subscribe(self, conn: AsyncConnection, symbol=None):
self.__reset(symbol=symbol)
for chan in set(self.channels or self.subscription):
await conn.write(
json.dumps({
"type":
"subscribe",
"product_ids":
list(self.symbols or self.subscription[chan]),
"channels": [chan]
}))
chan = feed_to_exchange(self.id, L3_BOOK)
if chan in set(self.channels or self.subscription):
await self._book_snapshot(
list(self.symbols or self.subscription[chan]))
async def subscribe(self, conn: AsyncConnection):
self.__reset()
client_id = 0
channels = []
for chan in self.subscription:
for pair in self.subscription[chan]:
channels.append(f"{chan}.{pair}.raw")
await conn.write(json.dumps(
{
"jsonrpc": "2.0",
"id": client_id,
"method": "public/subscribe",
"params": {
"channels": channels
}
}
))
async def subscribe(self, conn: AsyncConnection):
self.__reset()
# H: Hub, M: Message, A: Args, I: Internal ID
# For more signalR info see:
# https://blog.3d-logic.com/2015/03/29/signalr-on-the-wire-an-informal-description-of-the-signalr-protocol/
# http://blogs.microsoft.co.il/applisec/2014/03/12/signalr-message-format/
for chan in self.subscription:
channel = self.exchange_channel_to_std(chan)
i = 1
for symbol in self.subscription[chan]:
if channel == L2_BOOK:
msg = {
'A': ([chan.format(symbol, self.__depth())], ),
'H': 'c3',
'I': i,
'M': 'Subscribe'
}
elif channel in (TRADES, TICKER):
msg = {
'A': ([chan.format(symbol)], ),
'H': 'c3',
'I': i,
'M': 'Subscribe'
}
elif channel == CANDLES:
interval = None
if self.candle_interval == '1m':
interval = 'MINUTE_1'
elif self.candle_interval == '5m':
interval = 'MINUTE_5'
elif self.candle_interval == '1h':
interval = 'HOUR_1'
elif self.candle_interval == '1d':
interval = 'DAY_1'
msg = {
'A': ([chan.format(symbol, interval)], ),
'H': 'c3',
'I': i,
'M': 'Subscribe'
}
else:
LOG.error("%s: invalid subscription for channel %s",
channel)
await conn.write(json.dumps(msg))
i += 1
async def subscribe(self, connection: AsyncConnection):
self.__reset(connection)
for chan, symbols in connection.subscription.items():
if len(symbols) == 0:
continue
stype = str_to_symbol(
self.exchange_symbol_to_std_symbol(symbols[0])).type
msg = {
'type': 'subscribe',
'channels': [chan],
'instruments' if stype in {PERPETUAL, FUTURES, OPTION} else 'pairs':
symbols,
}
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
msg['token'] = self._auth_token
await connection.write(json.dumps(msg))
def generate_token(self, request: str, payload=None) -> dict:
if not payload:
payload = {}
payload['request'] = request
payload['nonce'] = int(time.time() * 1000)
if self.account_name:
payload['account'] = self.account_name
b64_payload = base64.b64encode(json.dumps(payload).encode('utf-8'))
signature = hmac.new(self.key_secret.encode('utf-8'), b64_payload,
hashlib.sha384).hexdigest()
return {
'X-GEMINI-PAYLOAD': b64_payload.decode(),
'X-GEMINI-APIKEY': self.key_id,
'X-GEMINI-SIGNATURE': signature
}
async def subscribe(self, conn: AsyncConnection, options: Tuple[str, List[str]] = None):
chan = options[0]
symbols = options[1]
sub = {"name": chan}
if 'book' in chan:
max_depth = self.max_depth if self.max_depth else 1000
if max_depth not in self.valid_depths:
for d in self.valid_depths:
if d > max_depth:
max_depth = d
sub['depth'] = max_depth
await conn.write(json.dumps({
"event": "subscribe",
"pair": symbols,
"subscription": sub
}))
async def subscribe(self, websocket):
self.websocket = websocket
self.__reset()
for chan in self.channels if self.channels else self.config:
if chan == FUNDING:
asyncio.create_task(self._funding(self.pairs if self.pairs else self.config[chan]))
continue
if chan == OPEN_INTEREST:
asyncio.create_task(self._open_interest(self.pairs if self.pairs else self.config[chan]))
continue
for pair in self.pairs if self.pairs else self.config[chan]:
await websocket.send(json.dumps(
{
"channel": chan,
"market": pair,
"op": "subscribe"
}
))
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
symbols = self.subscription[chan]
if chan == FUNDING:
asyncio.create_task(self._funding(symbols)) # TODO: use HTTPAsyncConn
continue
if chan == OPEN_INTEREST:
asyncio.create_task(self._open_interest(symbols)) # TODO: use HTTPAsyncConn
continue
for pair in symbols:
await conn.write(json.dumps(
{
"channel": chan,
"market": pair,
"op": "subscribe"
}
))
async def subscribe(self, websocket):
self.websocket = websocket
self.__reset()
client_id = 0
channels = []
for chan in self.channels if self.channels else self.config:
for pair in self.pairs if self.pairs else self.config[chan]:
channels.append(f"{chan}.{pair}.raw")
await websocket.send(
json.dumps({
"jsonrpc": "2.0",
"id": client_id,
"method": "public/subscribe",
"params": {
"channels": channels
}
}))
self.channel_map.update({c: {} for c in channels})
async def writer(self):
while True:
await self.connect()
async with self.read_queue() as update:
if self.conn_type == 'udp://':
if len(update) > self.mtu:
chunks = wrap(update, self.mtu)
for chunk in chunks:
msg = json.dumps({
'type': 'chunked',
'chunks': len(chunks),
'data': chunk
}).encode()
self.conn.sendto(msg)
else:
self.conn.sendto(update.encode())
else:
self.conn.write(update.encode())
async def generate_token(self, conn: AsyncConnection):
ts = int(time() * 1000)
msg = {
'op': 'login',
'args': {
'key':
self.key_id,
'sign':
hmac.new(self.key_secret.encode(),
f'{ts}websocket_login'.encode(),
'sha256').hexdigest(),
'time':
ts,
}
}
if self.subaccount:
msg['args']['subaccount'] = self.subaccount
await conn.write(json.dumps(msg))
async def test_load_multi_entity(conn):
user = User(
name={"family": "User"},
address=Address(addr="XYZ Addr"),
)
await conn.save(user)
other_field = raw(f"1").alias("something")
q = Query(User) \
.columns(User, other_field) \
.load(User.id, User.address.addr) \
.where(User.id == user.id)
user = await conn.select(q).first()
data = json.dumps(user)
# TODO: üres nem kívánt értékek törlése
assert data == """[{"id":8,"name":{},"address":{"addr":"XYZ Addr"},"children":[],"tags":[]},1]"""
async def message_handler(self, msg: str, timestamp: float):
# unzip message
msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS)
msg = json.loads(msg, parse_float=Decimal)
# Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it
if 'ping' in msg:
await self.websocket.send(json.dumps({'pong': msg['ping']}))
elif 'status' in msg and msg['status'] == 'ok':
return
elif 'ch' in msg:
if 'trade' in msg['ch']:
await self._trade(msg, timestamp)
elif 'depth' in msg['ch']:
await self._book(msg, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def subscribe(self, conn: AsyncConnection):
"""
Doc : https://docs.upbit.com/docs/upbit-quotation-websocket
For subscription, ticket information is commonly required.
In order to reduce the data size, format parameter is set to 'SIMPLE' instead of 'DEFAULT'
Examples (Note that the positions of the base and quote currencies are swapped.)
1. In order to get TRADES of "BTC-KRW" and "XRP-BTC" markets.
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC","BTC-XRP"]}]
2. In order to get ORDERBOOK of "BTC-KRW" and "XRP-BTC" markets.
> [{"ticket":"UNIQUE_TICKET"},{"type":"orderbook","codes":["KRW-BTC","BTC-XRP"]}]
3. In order to get TRADES of "BTC-KRW" and ORDERBOOK of "ETH-KRW"
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]}]
4. In order to get TRADES of "BTC-KRW", ORDERBOOK of "ETH-KRW and TICKER of "EOS-KRW"
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]},{"type":"ticker", "codes":["KRW-EOS"]}]
5. In order to get TRADES of "BTC-KRW", ORDERBOOK of "ETH-KRW and TICKER of "EOS-KRW" with in shorter format
> [{"ticket":"UNIQUE_TICKET"},{"format":"SIMPLE"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]},{"type":"ticker", "codes":["KRW-EOS"]}]
"""
chans = [{"ticket": uuid.uuid4()}, {"format": "SIMPLE"}]
for chan in self.subscription:
codes = list(self.subscription[chan])
if chan == L2_BOOK:
chans.append({
"type": "orderbook",
"codes": codes,
'isOnlyRealtime': True
})
if chan == TRADES:
chans.append({
"type": "trade",
"codes": codes,
'isOnlyRealtime': True
})
await conn.write(json.dumps(chans))
async def _login(self, conn: AsyncConnection):
LOG.debug("%s: Attempting authentication", conn.uuid)
timestamp = int(time())
msg = f"{timestamp}GET/user/verify"
msg = hmac.new(bytes(self.key_secret, encoding='utf8'),
bytes(msg, encoding='utf-8'),
digestmod='sha256')
sign = str(base64.b64encode(msg.digest()), 'utf8')
await conn.write(
json.dumps({
"op":
"login",
"args": [{
"apiKey": self.key_id,
"passphrase": self.key_passphrase,
"timestamp": timestamp,
"sign": sign
}]
}))
async def subscribe(self, conn: AsyncConnection, options: Tuple[str, List[str]] = None):
chan = options[0]
symbols = options[1]
sub = {"name": chan}
if normalize_channel(self.id, chan) == L2_BOOK:
max_depth = self.max_depth if self.max_depth else 1000
if max_depth not in self.valid_depths:
for d in self.valid_depths:
if d > max_depth:
max_depth = d
sub['depth'] = max_depth
if normalize_channel(self.id, chan) == CANDLES:
sub['interval'] = self.candle_interval
await conn.write(json.dumps({
"event": "subscribe",
"pair": symbols,
"subscription": sub
}))
async def _login(self, conn) -> None:
ts = int(time() * 1000)
await conn.send(
json.dumps({
'op': 'login',
'args': {
'key':
self.api_key,
'subaccount':
self.subaccount,
'sign':
hmac.new(self.api_secret.encode(),
f'{ts}websocket_login'.encode(),
'sha256').hexdigest(),
'time':
ts,
}
}))
self._logged_in = True
await asyncio.sleep(0)
async def subscribe(self, websocket):
"""
Doc : https://docs.upbit.com/docs/upbit-quotation-websocket
For subscription, ticket information is commonly required.
In order to reduce the data size, format parameter is set to 'SIMPLE' instead of 'DEFAULT'
Examples (Note that the positions of the base and quote currencies are swapped.)
1. In order to get TRADES of "BTC-KRW" and "XRP-BTC" markets.
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC","BTC-XRP"]}]
2. In order to get ORDERBOOK of "BTC-KRW" and "XRP-BTC" markets.
> [{"ticket":"UNIQUE_TICKET"},{"type":"orderbook","codes":["KRW-BTC","BTC-XRP"]}]
3. In order to get TRADES of "BTC-KRW" and ORDERBOOK of "ETH-KRW"
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]}]
4. In order to get TRADES of "BTC-KRW", ORDERBOOK of "ETH-KRW and TICKER of "EOS-KRW"
> [{"ticket":"UNIQUE_TICKET"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]},{"type":"ticker", "codes":["KRW-EOS"]}]
5. In order to get TRADES of "BTC-KRW", ORDERBOOK of "ETH-KRW and TICKER of "EOS-KRW" with in shorter format
> [{"ticket":"UNIQUE_TICKET"},{"format":"SIMPLE"},{"type":"trade","codes":["KRW-BTC"]},{"type":"orderbook","codes":["KRW-ETH"]},{"type":"ticker", "codes":["KRW-EOS"]}]
"""
self.__reset()
chans = [{"ticket": "UNIQUE_TICKET"}, {"format": "SIMPLE"}]
for channel in self.channels if not self.config else self.config:
codes = list()
for pair in self.pairs if not self.config else self.config[channel]:
codes.append(pair)
if channel == L2_BOOK:
chans.append({"type": "orderbook", "codes": codes})
if channel == TRADES:
chans.append({"type": "trade", "codes": codes})
if channel == TICKER:
chans.append({"type": "ticker", "codes": codes})
await websocket.send(json.dumps(chans))
async def subscribe(self, conn: AsyncConnection):
if self.key_id and self.key_passphrase and self.key_secret:
await self._login(conn)
self.__reset(conn)
args = []
interval = self.candle_interval
if interval[-1] != 'm':
interval[-1] = interval[-1].upper()
for chan, symbols in conn.subscription.items():
for s in symbols:
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(s))
if sym.type == SPOT:
if chan == 'positions': # positions not applicable on spot
continue
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
itype = 'spbl'
s += '_SPBL'
else:
itype = 'SP'
else:
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
itype = s.split('_')[-1]
if chan == 'orders':
s = 'default' # currently only supports 'default' for order channel on futures
else:
itype = 'MC'
s = s.split("_")[0]
d = {
'instType': itype,
'channel':
chan if chan != 'candle' else 'candle' + interval,
'instId': s
}
args.append(d)
await conn.write(json.dumps({"op": "subscribe", "args": args}))
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan, symbols in conn.subscription.items():
sub = {"name": chan}
if self.exchange_channel_to_std(chan) == L2_BOOK:
max_depth = self.max_depth if self.max_depth else 1000
if max_depth not in self.valid_depths:
for d in self.valid_depths:
if d > max_depth:
max_depth = d
break
sub['depth'] = max_depth
if self.exchange_channel_to_std(chan) == CANDLES:
sub['interval'] = self.candle_interval_map[self.candle_interval]
await conn.write(json.dumps({
"event": "subscribe",
"pair": symbols,
"subscription": sub
}))
async def _login(self, conn: AsyncConnection):
# will return error if already logged in
if not self._logged_in:
ts = int(time() * 1000)
await conn.send(
json.dumps({
'op': 'login',
'args': {
'key':
self.key_id,
'sign':
hmac.new(self.key_secret.encode(),
f'{ts}websocket_login'.encode(),
'sha256').hexdigest(),
'time':
ts,
'subaccount':
self.config[self.id.lower()].get('subaccount')
}
}))
self._logged_in = True
async def subscribe(self, conn: AsyncConnection):
self.__reset()
for chan in self.subscription:
if not self.is_authenticated_channel(chan):
if chan == LIQUIDATIONS:
continue
for symbol in self.subscription[chan]:
sym = self.exchange_symbol_to_std_symbol(symbol)
instrument_type = self.instrument_type(sym)
if instrument_type != PERPETUAL and 'funding' in chan:
continue # No funding for spot, futures and options
if instrument_type == SPOT and chan == 'open-interest':
continue # No open interest for spot
request = {
"op": "subscribe",
"args": [{
"channel": chan,
"instId": symbol
}]
}
await conn.write(json.dumps(request))
async def subscribe(self, conn: AsyncConnection):
self.__reset()
args = []
interval = self.candle_interval
if interval[-1] != 'm':
interval[-1] = interval[-1].upper()
for chan, symbols in conn.subscription.items():
for s in symbols:
d = {
'instType':
'SPBL' if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)) else 'SP',
'channel':
chan if chan != 'candle' else 'candle' + interval,
'instId':
s
}
args.append(d)
await conn.write(json.dumps({"op": "subscribe", "args": args}))
async def _request(self,
method: str,
endpoint: str,
auth: bool = False,
body=None,
retry_count=1,
retry_delay=60):
api = self.sandbox_api if self.sandbox else self.api
header = None
if auth:
header = self._generate_signature(
endpoint, method, body=json.dumps(body) if body else '')
if method == "GET":
data = await self.http_conn.read(f'{api}{endpoint}', header=header)
elif method == 'POST':
data = await self.http_conn.write(f'{api}{endpoint}',
msg=body,
header=header)
elif method == 'DELETE':
data = await self.http_conn.delete(f'{api}{endpoint}',
header=header)
return json.loads(data, parse_float=Decimal)
async def _authenticate(self, conn: AsyncConnection):
"""Send API Key with signed message."""
# Docs: https://www.bitmex.com/app/apiKeys
# https://github.com/BitMEX/sample-market-maker/blob/master/test/websocket-apikey-auth-test.py
config = self.config[self.id.lower()]
key_id = os.environ.get('CF_BITMEX_KEY_ID') or config.key_id
key_secret = os.environ.get(
'CF_BITMEX_KEY_SECRET') or config.key_secret
if key_id and key_secret:
LOG.info('%s: Authenticate with signature', conn.uuid)
expires = int(time.time()) + 365 * 24 * 3600 # One year
msg = f'GET/realtime{expires}'.encode('utf-8')
signature = hmac.new(key_secret.encode('utf-8'),
msg,
digestmod=hashlib.sha256).hexdigest()
await conn.write(
json.dumps({
'op': 'authKeyExpires',
'args': [key_id, expires, signature]
}))
else:
LOG.info(
'%s: No authentication. Enable it using config or env. vars: CF_BITMEX_KEY_ID and CF_BITMEX_KEY_SECRET',
conn.uuid)
async def authenticate(self, conn: AsyncConnection):
if self.requires_authentication:
# https://api.bequant.io/#socket-session-authentication
# Nonce should be random string
nonce = 'h'.join(
random.choices(string.ascii_letters + string.digits,
k=16)).encode('utf-8')
signature = hmac.new(self.key_secret.encode('utf-8'), nonce,
hashlib.sha256).hexdigest()
auth = {
"method": "login",
"params": {
"algo": "HS256",
"pKey": self.key_id,
"nonce": nonce.decode(),
"signature": signature
},
"id": conn.uuid
}
await conn.write(json.dumps(auth))
LOG.debug(f"{conn.uuid}: Authenticating with message: {auth}")
return conn