def subscription_filter(self, sub: dict) -> dict:
if not self.instrument_filter and not self.channel_filter:
return sub
ret = {}
for chan, syms in sub.items():
if self.channel_filter and chan not in self.channel_filter:
continue
ret[chan] = []
if not self.instrument_filter:
ret[chan].extend(sub[chan])
else:
if self.instrument_filter[0] == 'TYPE':
ret[chan].extend([
s for s in syms
if str_to_symbol(s).type in self.instrument_filter[1]
])
elif self.instrument_filter[0] == 'QUOTE':
ret[chan].extend([
s for s in syms
if str_to_symbol(s).quote in self.instrument_filter[1]
])
else:
raise ValueError(
'Invalid instrument filter type specified')
return ret
async def subscribe(self, conn: AsyncConnection):
self.__reset()
args = []
interval = self.candle_interval
if interval[-1] != 'm':
interval[-1] = interval[-1].upper()
for chan, symbols in conn.subscription.items():
for s in symbols:
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(s))
if sym.type == SPOT:
itype = 'SPBL' if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)) else 'SP'
else:
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
itype = s.split('_')[-1]
else:
itype = 'MC'
s = s.split("_")[0]
d = {
'instType': itype,
'channel':
chan if chan != 'candle' else 'candle' + interval,
'instId': s
}
args.append(d)
await conn.write(json.dumps({"op": "subscribe", "args": args}))
async def subscribe(self, connection: AsyncConnection):
self.__reset()
for chan in connection.subscription:
if not self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
for pair in connection.subscription[chan]:
sym = str_to_symbol(
self.exchange_symbol_to_std_symbol(pair))
if self.exchange_channel_to_std(chan) == CANDLES:
c = chan if sym.quote == 'USD' else 'candle'
sub = [
f"{c}.{self.candle_interval_map[self.candle_interval]}.{pair}"
]
else:
sub = [f"{chan}.{pair}"]
await connection.write(
json.dumps({
"op": "subscribe",
"args": sub
}))
else:
await connection.write(
json.dumps({
"op": "subscribe",
"args": [f"{chan}"]
}))
async def authenticate(self, connection: AsyncConnection):
if not self.key_id or not self.key_secret:
return
if any([
self.is_authenticated_channel(self.exchange_channel_to_std(c))
for c in connection.subscription
]):
symbols = list(
set(itertools.chain(*connection.subscription.values())))
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(symbols[0]))
for ep in self.rest_endpoints:
if sym.type in ep.instrument_filter[1]:
ts = int(round(time.time() * 1000))
signature = self.get_signature(ep.routes.authentication,
{'timestamp': ts})
params = {'timestamp': ts, 'signature': signature}
ret = self.http_sync.read(
ep.route('authentication', sandbox=self.sandbox),
params=params,
headers={'X-Bit-Access-Key': self.key_id},
json=True)
if ret['code'] != 0 or 'token' not in ret['data']:
LOG.warning('%s: authentication failed: %s', ret)
token = ret['data']['token']
self._auth_token = token
return
def write(self, exchange, data_type, pair, timestamp):
if self.data.empty:
# TODO log?
# TODO add dummy file? If not some alerts on s3 may fire as no data is written in a fixed timeframe
# TODO there is a metric upstream (in aggregator) tracking block size, writing 0 if empty
return
df = self.data
df['version'] = self.version
df['date'] = pd.to_datetime(df['receipt_timestamp'],
unit='s').dt.strftime('%Y-%m-%d')
first_timestamp = df['receipt_timestamp'].iloc[0]
last_timestamp = df['receipt_timestamp'].iloc[-1]
df['exchange'] = exchange
df['symbol'] = pair # TODO rename pair to symbol
# parse base, quote and instrument type
symbol = str_to_symbol(pair)
df['quote'] = symbol.quote
df['base'] = symbol.base
df['instrument_type'] = symbol.type
df['instrument_extra'] = json.dumps({}) # TODO instrument_extra
# indicate as raw uncompacted data
df['compaction'] = 'raw'
# TODO check if date > 1 (i.e. this block has two days overlap) and make separate writes
# TODO or check if wrangler already does it
wr.s3.to_parquet(df=df,
path=f's3://{self.bucket}/{self.prefix}/{data_type}',
compression=self.compression,
dataset=True,
filename_prefix=self._filename_prefix(
exchange, data_type, pair, first_timestamp,
last_timestamp),
partition_cols=[
'exchange',
'instrument_type',
'instrument_extra',
'symbol',
'base',
'quote',
'date',
'compaction',
'version',
],
schema_evolution=True,
database=self.database,
table=self._athena_table_name(data_type),
boto3_session=self.boto3_session)
async def _funding(self, pairs):
"""
{
"status": "ok",
"data": {
"estimated_rate": "0.000100000000000000",
"funding_rate": "-0.000362360011416593",
"contract_code": "BTC-USD",
"symbol": "BTC",
"fee_asset": "BTC",
"funding_time": "1603872000000",
"next_funding_time": "1603900800000"
},
"ts": 1603866304635
}
"""
while True:
for pair in pairs:
# use symbol to look up correct endpoint
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(pair))
endpoint = None
for ep in self.rest_endpoints:
if sym.quote in ep.instrument_filter[1]:
endpoint = self.rest_endpoints[0].route(
'funding').format(pair)
data = await self.http_conn.read(endpoint)
data = json.loads(data, parse_float=Decimal)
received = time.time()
update = (data['data']['funding_rate'],
self.timestamp_normalize(
int(data['data']['next_funding_time'])))
if pair in self.funding_updates and self.funding_updates[
pair] == update:
await asyncio.sleep(1)
continue
self.funding_updates[pair] = update
f = Funding(self.id,
self.exchange_symbol_to_std_symbol(pair),
None,
Decimal(data['data']['funding_rate']),
self.timestamp_normalize(
int(data['data']['next_funding_time'])),
self.timestamp_normalize(
int(data['data']['funding_time'])),
predicted_rate=Decimal(
data['data']['estimated_rate']),
raw=data)
await self.callback(FUNDING, f, received)
await asyncio.sleep(0.1)
async def subscribe(self, connection: AsyncConnection):
self.__reset(connection)
for chan, symbols in connection.subscription.items():
if len(symbols) == 0:
continue
stype = str_to_symbol(
self.exchange_symbol_to_std_symbol(symbols[0])).type
msg = {
'type': 'subscribe',
'channels': [chan],
'instruments' if stype in {PERPETUAL, FUTURES, OPTION} else 'pairs':
symbols,
}
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
msg['token'] = self._auth_token
await connection.write(json.dumps(msg))
async def subscribe(self, conn: AsyncConnection):
if self.key_id and self.key_passphrase and self.key_secret:
await self._login(conn)
self.__reset(conn)
args = []
interval = self.candle_interval
if interval[-1] != 'm':
interval[-1] = interval[-1].upper()
for chan, symbols in conn.subscription.items():
for s in symbols:
sym = str_to_symbol(self.exchange_symbol_to_std_symbol(s))
if sym.type == SPOT:
if chan == 'positions': # positions not applicable on spot
continue
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
itype = 'spbl'
s += '_SPBL'
else:
itype = 'SP'
else:
if self.is_authenticated_channel(
self.exchange_channel_to_std(chan)):
itype = s.split('_')[-1]
if chan == 'orders':
s = 'default' # currently only supports 'default' for order channel on futures
else:
itype = 'MC'
s = s.split("_")[0]
d = {
'instType': itype,
'channel':
chan if chan != 'candle' else 'candle' + interval,
'instId': s
}
args.append(d)
await conn.write(json.dumps({"op": "subscribe", "args": args}))