def subscription_filter(self, sub: dict) -> dict: if not self.instrument_filter and not self.channel_filter: return sub ret = {} for chan, syms in sub.items(): if self.channel_filter and chan not in self.channel_filter: continue ret[chan] = [] if not self.instrument_filter: ret[chan].extend(sub[chan]) else: if self.instrument_filter[0] == 'TYPE': ret[chan].extend([ s for s in syms if str_to_symbol(s).type in self.instrument_filter[1] ]) elif self.instrument_filter[0] == 'QUOTE': ret[chan].extend([ s for s in syms if str_to_symbol(s).quote in self.instrument_filter[1] ]) else: raise ValueError( 'Invalid instrument filter type specified') return ret
async def subscribe(self, conn: AsyncConnection): self.__reset() args = [] interval = self.candle_interval if interval[-1] != 'm': interval[-1] = interval[-1].upper() for chan, symbols in conn.subscription.items(): for s in symbols: sym = str_to_symbol(self.exchange_symbol_to_std_symbol(s)) if sym.type == SPOT: itype = 'SPBL' if self.is_authenticated_channel( self.exchange_channel_to_std(chan)) else 'SP' else: if self.is_authenticated_channel( self.exchange_channel_to_std(chan)): itype = s.split('_')[-1] else: itype = 'MC' s = s.split("_")[0] d = { 'instType': itype, 'channel': chan if chan != 'candle' else 'candle' + interval, 'instId': s } args.append(d) await conn.write(json.dumps({"op": "subscribe", "args": args}))
async def subscribe(self, connection: AsyncConnection): self.__reset() for chan in connection.subscription: if not self.is_authenticated_channel( self.exchange_channel_to_std(chan)): for pair in connection.subscription[chan]: sym = str_to_symbol( self.exchange_symbol_to_std_symbol(pair)) if self.exchange_channel_to_std(chan) == CANDLES: c = chan if sym.quote == 'USD' else 'candle' sub = [ f"{c}.{self.candle_interval_map[self.candle_interval]}.{pair}" ] else: sub = [f"{chan}.{pair}"] await connection.write( json.dumps({ "op": "subscribe", "args": sub })) else: await connection.write( json.dumps({ "op": "subscribe", "args": [f"{chan}"] }))
async def authenticate(self, connection: AsyncConnection): if not self.key_id or not self.key_secret: return if any([ self.is_authenticated_channel(self.exchange_channel_to_std(c)) for c in connection.subscription ]): symbols = list( set(itertools.chain(*connection.subscription.values()))) sym = str_to_symbol(self.exchange_symbol_to_std_symbol(symbols[0])) for ep in self.rest_endpoints: if sym.type in ep.instrument_filter[1]: ts = int(round(time.time() * 1000)) signature = self.get_signature(ep.routes.authentication, {'timestamp': ts}) params = {'timestamp': ts, 'signature': signature} ret = self.http_sync.read( ep.route('authentication', sandbox=self.sandbox), params=params, headers={'X-Bit-Access-Key': self.key_id}, json=True) if ret['code'] != 0 or 'token' not in ret['data']: LOG.warning('%s: authentication failed: %s', ret) token = ret['data']['token'] self._auth_token = token return
def write(self, exchange, data_type, pair, timestamp): if self.data.empty: # TODO log? # TODO add dummy file? If not some alerts on s3 may fire as no data is written in a fixed timeframe # TODO there is a metric upstream (in aggregator) tracking block size, writing 0 if empty return df = self.data df['version'] = self.version df['date'] = pd.to_datetime(df['receipt_timestamp'], unit='s').dt.strftime('%Y-%m-%d') first_timestamp = df['receipt_timestamp'].iloc[0] last_timestamp = df['receipt_timestamp'].iloc[-1] df['exchange'] = exchange df['symbol'] = pair # TODO rename pair to symbol # parse base, quote and instrument type symbol = str_to_symbol(pair) df['quote'] = symbol.quote df['base'] = symbol.base df['instrument_type'] = symbol.type df['instrument_extra'] = json.dumps({}) # TODO instrument_extra # indicate as raw uncompacted data df['compaction'] = 'raw' # TODO check if date > 1 (i.e. this block has two days overlap) and make separate writes # TODO or check if wrangler already does it wr.s3.to_parquet(df=df, path=f's3://{self.bucket}/{self.prefix}/{data_type}', compression=self.compression, dataset=True, filename_prefix=self._filename_prefix( exchange, data_type, pair, first_timestamp, last_timestamp), partition_cols=[ 'exchange', 'instrument_type', 'instrument_extra', 'symbol', 'base', 'quote', 'date', 'compaction', 'version', ], schema_evolution=True, database=self.database, table=self._athena_table_name(data_type), boto3_session=self.boto3_session)
async def _funding(self, pairs): """ { "status": "ok", "data": { "estimated_rate": "0.000100000000000000", "funding_rate": "-0.000362360011416593", "contract_code": "BTC-USD", "symbol": "BTC", "fee_asset": "BTC", "funding_time": "1603872000000", "next_funding_time": "1603900800000" }, "ts": 1603866304635 } """ while True: for pair in pairs: # use symbol to look up correct endpoint sym = str_to_symbol(self.exchange_symbol_to_std_symbol(pair)) endpoint = None for ep in self.rest_endpoints: if sym.quote in ep.instrument_filter[1]: endpoint = self.rest_endpoints[0].route( 'funding').format(pair) data = await self.http_conn.read(endpoint) data = json.loads(data, parse_float=Decimal) received = time.time() update = (data['data']['funding_rate'], self.timestamp_normalize( int(data['data']['next_funding_time']))) if pair in self.funding_updates and self.funding_updates[ pair] == update: await asyncio.sleep(1) continue self.funding_updates[pair] = update f = Funding(self.id, self.exchange_symbol_to_std_symbol(pair), None, Decimal(data['data']['funding_rate']), self.timestamp_normalize( int(data['data']['next_funding_time'])), self.timestamp_normalize( int(data['data']['funding_time'])), predicted_rate=Decimal( data['data']['estimated_rate']), raw=data) await self.callback(FUNDING, f, received) await asyncio.sleep(0.1)
async def subscribe(self, connection: AsyncConnection): self.__reset(connection) for chan, symbols in connection.subscription.items(): if len(symbols) == 0: continue stype = str_to_symbol( self.exchange_symbol_to_std_symbol(symbols[0])).type msg = { 'type': 'subscribe', 'channels': [chan], 'instruments' if stype in {PERPETUAL, FUTURES, OPTION} else 'pairs': symbols, } if self.is_authenticated_channel( self.exchange_channel_to_std(chan)): msg['token'] = self._auth_token await connection.write(json.dumps(msg))
async def subscribe(self, conn: AsyncConnection): if self.key_id and self.key_passphrase and self.key_secret: await self._login(conn) self.__reset(conn) args = [] interval = self.candle_interval if interval[-1] != 'm': interval[-1] = interval[-1].upper() for chan, symbols in conn.subscription.items(): for s in symbols: sym = str_to_symbol(self.exchange_symbol_to_std_symbol(s)) if sym.type == SPOT: if chan == 'positions': # positions not applicable on spot continue if self.is_authenticated_channel( self.exchange_channel_to_std(chan)): itype = 'spbl' s += '_SPBL' else: itype = 'SP' else: if self.is_authenticated_channel( self.exchange_channel_to_std(chan)): itype = s.split('_')[-1] if chan == 'orders': s = 'default' # currently only supports 'default' for order channel on futures else: itype = 'MC' s = s.split("_")[0] d = { 'instType': itype, 'channel': chan if chan != 'candle' else 'candle' + interval, 'instId': s } args.append(d) await conn.write(json.dumps({"op": "subscribe", "args": args}))