def test_poloniex_pair_conversions(): for _, pair in poloniex_pair_mapping.items(): std = pair_exchange_to_std(pair) assert (pair == pair_std_to_exchange(std, POLONIEX))
def get_active_symbols(): load_exchange_pair_mapping(Upbit.id) symbols = [] for data in Upbit.get_active_symbols_info(): symbols.append(pair_exchange_to_std(data['market'])) return symbols
def test_coinbase_pair_conversions(): for _, pair in coinbase_pair_mapping.items(): assert (pair_exchange_to_std(pair) == pair_std_to_exchange( pair, COINBASE))
async def _book(self, msg: dict, chan_id: int, timestamp: float): delta = {BID: [], ASK: []} msg_type = msg[0][0] pair = None forced = False # initial update (i.e. snapshot) if msg_type == 'i': forced = True pair = msg[0][1]['currencyPair'] pair = pair_exchange_to_std(pair) self.l2_book[pair] = {BID: sd(), ASK: sd()} # 0 is asks, 1 is bids order_book = msg[0][1]['orderBook'] for key in order_book[0]: amount = Decimal(order_book[0][key]) price = Decimal(key) self.l2_book[pair][ASK][price] = amount for key in order_book[1]: amount = Decimal(order_book[1][key]) price = Decimal(key) self.l2_book[pair][BID][price] = amount else: pair = self.pair_mapping[chan_id] pair = pair_exchange_to_std(pair) for update in msg: msg_type = update[0] # order book update if msg_type == 'o': side = ASK if update[1] == 0 else BID price = Decimal(update[2]) amount = Decimal(update[3]) if amount == 0: delta[side].append((price, 0)) del self.l2_book[pair][side][price] else: delta[side].append((price, amount)) self.l2_book[pair][side][price] = amount elif msg_type == 't': # index 1 is trade id, 2 is side, 3 is price, 4 is amount, 5 is timestamp _, order_id, _, price, amount, server_ts = update price = Decimal(price) amount = Decimal(amount) side = BUY if update[2] == 1 else SELL if self.__do_callback(TRADES, pair): await self.callback(TRADES, feed=self.id, pair=pair, side=side, amount=amount, price=price, timestamp=float(server_ts), order_id=order_id, receipt_timestamp=timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg) if self.__do_callback(L2_BOOK, pair): await self.book_callback(self.l2_book[pair], L2_BOOK, pair, forced, delta, timestamp, timestamp)
async def _book_update(self, msg: dict, timestamp: float): """ Snapshot: [ [ 'AE', '1', 'BTC_USDT', '1547941504', { 'asks':[ [ '25000.00000000', '0.02000000' ], [ '19745.83000000', '0.00200000' ], [ '19698.96000000', '0.00100000' ], ... ] }, { 'bids':[ [ '3662.83040000', '0.00100000' ], [ '3662.77540000', '0.01000000' ], [ '3662.59900000', '0.10300000' ], ... ] } ] ] Update: ['E', '1', '1547942636', 'BTC_USDT', 'ASK', '3674.91740000', '0.02600000'] """ forced = False delta = {BID: [], ASK: []} if msg[0] == 'AE': # snapshot forced = True pair = pair_exchange_to_std(msg[2]) ts = msg[3] asks = msg[4]['asks'] if 'asks' in msg[4] else msg[5]['asks'] bids = msg[5]['bids'] if 'bids' in msg[5] else msg[4]['bids'] self.l2_book[pair] = { BID: sd({Decimal(price): Decimal(amount) for price, amount in bids}), ASK: sd({Decimal(price): Decimal(amount) for price, amount in asks}) } else: # Update ts = msg[2] pair = pair_exchange_to_std(msg[3]) side = ASK if msg[4] == 'ASK' else BID price = Decimal(msg[5]) amount = Decimal(msg[6]) if amount == 0: if price in self.l2_book[pair][side]: del self.l2_book[pair][side][price] delta[side].append((price, 0)) else: self.l2_book[pair][side][price] = amount delta[side].append((price, amount)) await self.book_callback(self.l2_book[pair], L2_BOOK, pair, forced, delta, ts, timestamp)
def test_gemini_pair_conversions(): for _, pair in gemini_pair_mapping.items(): std = pair_exchange_to_std(pair) assert (pair == pair_std_to_exchange(std, GEMINI))
async def _raw_book(self, msg): chan_id = msg[0] pair = self.channel_map[chan_id]['symbol'] pair = pair_exchange_to_std(pair) if isinstance(msg[1], list): if isinstance(msg[1][0], list): # snapshot so clear book self.l2_book[pair] = {BID: sd(), ASK: sd()} for update in msg[1]: order_id, price, amount = update price = Decimal(price) amount = Decimal(amount) if amount > 0: side = BID else: side = ASK amount = abs(amount) if price not in self.l2_book[pair][side]: self.l2_book[pair][side][price] = amount self.order_map[order_id] = { 'price': price, 'amount': amount, 'side': side } else: self.l2_book[pair][side][price] self.l2_book[pair][side][price] += amount self.order_map[order_id] = { 'price': price, 'amount': amount, 'side': side } else: # book update order_id, price, amount = [Decimal(x) for x in msg[1]] if amount > 0: side = BID else: side = ASK amount = abs(amount) if price == 0: price = self.order_map[order_id]['price'] self.l2_book[pair][side][price] -= self.order_map[ order_id]['amount'] if self.l2_book[pair][side][price] == 0: del self.l2_book[pair][side][price] del self.order_map[order_id] else: self.order_map[order_id] = { 'price': price, 'amount': amount, 'side': side } if price in self.l2_book[pair][side]: self.l2_book[pair][side][price] self.l2_book[pair][side][price] += amount else: self.l2_book[pair][side][price] = amount elif msg[1] == 'hb': pass else: LOG.warning("{} - Unexpected book msg {}".format(self.id, msg)) if L3_BOOK in self.standardized_channels: await self.callbacks[L3_BOOK](feed=self.id, pair=pair, book=self.l2_book[pair]) else: await self.callbacks[L2_BOOK](feed=self.id, pair=pair, book=self.l2_book[pair])
async def _book(self, msg, chan_id, sequence): msg_type = msg[0][0] pair = None # initial update (i.e. snapshot) if msg_type == 'i': pair = msg[0][1]['currencyPair'] pair = pair_exchange_to_std(pair) self.l3_book[pair] = {BID: sd(), ASK: sd()} # 0 is asks, 1 is bids order_book = msg[0][1]['orderBook'] for key in order_book[0]: amount = Decimal(order_book[0][key]) price = Decimal(key) self.l3_book[pair][ASK][price] = amount for key in order_book[1]: amount = Decimal(order_book[1][key]) price = Decimal(key) self.l3_book[pair][BID][price] = amount else: pair = poloniex_id_pair_mapping[chan_id] pair = pair_exchange_to_std(pair) for update in msg: timestamp = None msg_type = update[0] # order book update if msg_type == 'o': mtype = 'change' side = ASK if update[1] == 0 else BID price = Decimal(update[2]) amount = Decimal(update[3]) if amount == 0: del self.l3_book[pair][side][price] else: self.l3_book[pair][side][price] = amount elif msg_type == 't': # index 1 is trade id, 2 is side, 3 is price, 4 is amount, 5 is timestamp mtype = 'trade' timestamp = self.tz_aware_datetime_from_string(update[5]) price = Decimal(update[3]) side = ASK if update[2] == 0 else BID amount = Decimal(update[4]) await self.callbacks[TRADES](feed=self.id, pair=pair, side=side, amount=amount, price=price) else: LOG.warning("{} - Unexpected message received: {}".format(self.id, msg)) # continue so we don't hit the callback below if the msg_type isn't of the 2 tested for above continue await self.callbacks[L3_BOOK_UPDATE](feed=self.id, pair=pair, msg_type=mtype, timestamp=timestamp, sequence=sequence, side=side, price=price, size=amount) await self.callbacks[L3_BOOK](feed=self.id, sequence=sequence, timestamp=None, pair=pair, book=self.l3_book[pair])
def test_dsx_pair_conversion(): load_exchange_pair_mapping(DSX) for _, pair in dsx_pairs().items(): std = pair_exchange_to_std(pair) assert (pair == pair_std_to_exchange(std, DSX))
async def _book_update(self, msg): """ Snapshot: [ [ 'AE', '1', 'BTC_USDT', '1547941504', { 'asks':[ [ '25000.00000000', '0.02000000' ], [ '19745.83000000', '0.00200000' ], [ '19698.96000000', '0.00100000' ], ... ] }, { 'bids':[ [ '3662.83040000', '0.00100000' ], [ '3662.77540000', '0.01000000' ], [ '3662.59900000', '0.10300000' ], ... ] } ] ] Update: ['E', '1', '1547942636', 'BTC_USDT', 'ASK', '3674.91740000', '0.02600000'] """ if msg[0] == 'AE': # snapshot pair = pair_exchange_to_std(msg[2]) timestamp = msg[3] asks = msg[4]['asks'] if 'asks' in msg[4] else msg[5]['asks'] bids = msg[5]['bids'] if 'bids' in msg[5] else msg[4]['bids'] self.l2_book[pair] = { BID: sd({ Decimal(price): Decimal(amount) for price, amount in bids }), ASK: sd({ Decimal(price): Decimal(amount) for price, amount in asks }) } else: # Update timestamp = msg[2] pair = pair_exchange_to_std(msg[3]) side = ASK if msg[4] == 'ASK' else BID price = Decimal(msg[5]) amount = Decimal(msg[6]) if amount == 0: try: del self.l2_book[pair][side][price] except BaseException: pass else: self.l2_book[pair][side][price] = amount await self.book_callback(pair, L2_BOOK, True, None, timestamp)
async def _ticker(self, msg): await self.callbacks[TICKER](feed=self.id, pair=pair_exchange_to_std(msg['symbol']), bid=Decimal(msg['bid']), ask=Decimal(msg['ask']))
async def ticker(self, msg: dict, timestamp: float): for t in msg['D']: if (not self.config and t['M'] in self.pairs) or ('SubscribeToSummaryDeltas' in self.config and t['M'] in self.config['SubscribeToSummaryDeltas']): await self.callback(TICKER, feed=self.id, pair=pair_exchange_to_std(t['M']), bid=Decimal(t['B']), ask=Decimal(t['A']), timestamp=timestamp_normalize(self.id, t['T']), receipt_timestamp=timestamp)
def test_blockchain_pair_conversions(): load_exchange_pair_mapping(BLOCKCHAIN) for _, pair in blockchain_pairs().items(): assert (pair_exchange_to_std(pair) == pair_std_to_exchange( pair, BLOCKCHAIN))
async def _ticker(self, msg): await self.callback(TICKER, feed=self.id, pair=pair_exchange_to_std(msg['symbol']), bid=Decimal(msg['bid']), ask=Decimal(msg['ask']), timestamp=timestamp_normalize(self.id, msg['timestamp']))
def test_bitfinex_pair_conversions(): for _, pair in bitfinex_pair_mapping.items(): std = pair_exchange_to_std(pair) assert (pair == pair_std_to_exchange(std, BITFINEX))
def test_bitcoincom_pair_conversions(): load_exchange_pair_mapping(BITCOINCOM) for _, pair in bitcoincom_pairs().items(): std = pair_exchange_to_std(pair) assert (pair == pair_std_to_exchange(std, BITCOINCOM))
def test_hitbtc_pair_conversions(): for _, pair in hitbtc_pair_mapping.items(): std = pair_exchange_to_std(pair) assert (pair == pair_std_to_exchange(std, HITBTC))
async def _l2_update(self, msg: dict, timestamp: float): ''' { "channel":"marketdata", "market_id":"ETH-BTC", "status":"ok", "lag":0, "order_books":[ { "side":"buy", "price":"0.0165", "quantity":"0.47" },{ "side":"buy", "price":"0", "quantity":"14656.177" },{ "side":"sell", "price":"6400", "quantity":"0.001" }], "reset":true } { "channel":"marketdata", "market_id":"ETH-BTC", "status":"ok", "lag":0, "order_books":[ { "side":"buy", "price":"0.0281", "quantity":"48.541" },{ "side":"sell", "price":"0.0283", "quantity":"0" }] } ''' pair = pair_exchange_to_std(msg['market_id']) is_snapshot = msg.get('reset', False) if is_snapshot: self.l2_book[pair] = {ASK: sd(), BID: sd()} for entry in msg["order_books"]: price = Decimal(entry['price']) quantity = Decimal(entry['quantity']) side = BID if entry['side'] == "buy" else ASK self.l2_book[pair][side][price] = quantity await self.book_callback(self.l2_book[pair], L2_BOOK, pair, True, None, timestamp, timestamp) else: delta = {BID: [], ASK: []} for entry in msg["order_books"]: price = Decimal(entry['price']) quantity = Decimal(entry['quantity']) side = BID if entry['side'] == "buy" else ASK if quantity == 0: if price in self.l2_book[pair][side]: del self.l2_book[pair][side][price] delta[side].append((price, 0)) else: self.l2_book[pair][side][price] = quantity delta[side].append((price, quantity)) await self.book_callback(self.l2_book[pair], L2_BOOK, pair, False, delta, timestamp, timestamp)
def test_bitstamp_pair_conversions(): for _, pair in bitstamp_pair_mapping.items(): std = pair_exchange_to_std(pair) assert (pair == pair_std_to_exchange(std, BITSTAMP))
def __reset(self, pairs): for pair in pairs: self.l2_book[pair_exchange_to_std(pair)] = {BID: sd(), ASK: sd()}
async def _raw_book(self, msg: dict, timestamp: float): """ For L3 book updates """ def add_to_book(pair, side, price, order_id, amount): if price in self.l3_book[pair][side]: self.l3_book[pair][side][price][order_id] = amount else: self.l3_book[pair][side][price] = {order_id: amount} def remove_from_book(pair, side, order_id): price = self.order_map[pair][side][order_id]['price'] del self.l3_book[pair][side][price][order_id] if len(self.l3_book[pair][side][price]) == 0: del self.l3_book[pair][side][price] delta = {BID: [], ASK: []} forced = False chan_id = msg[0] pair = self.channel_map[chan_id]['symbol'] pair = pair_exchange_to_std(pair) if isinstance(msg[1], list): if isinstance(msg[1][0], list): # snapshot so clear orders self.order_map[pair] = {BID: {}, ASK: {}} self.l3_book[pair] = {BID: sd(), ASK: sd()} for update in msg[1]: order_id, price, amount = update price = Decimal(price) amount = Decimal(amount) if amount > 0: side = BID else: side = ASK amount = abs(amount) self.order_map[pair][side][order_id] = { 'price': price, 'amount': amount } add_to_book(pair, side, price, order_id, amount) forced = True else: # book update order_id, price, amount = msg[1] price = Decimal(price) amount = Decimal(amount) if amount > 0: side = BID else: side = ASK amount = abs(amount) if price == 0: price = self.order_map[pair][side][order_id]['price'] remove_from_book(pair, side, order_id) del self.order_map[pair][side][order_id] delta[side].append((order_id, price, 0)) else: if order_id in self.order_map[pair][side]: del_price = self.order_map[pair][side][order_id][ 'price'] delta[side].append((order_id, del_price, 0)) # remove existing order before adding new one delta[side].append((order_id, price, amount)) remove_from_book(pair, side, order_id) else: delta[side].append((order_id, price, amount)) add_to_book(pair, side, price, order_id, amount) self.order_map[pair][side][order_id] = { 'price': price, 'amount': amount } elif msg[1] == 'hb': return else: LOG.warning("%s: Unexpected book msg %s", self.id, msg) return await self.book_callback(self.l3_book[pair], L3_BOOK, pair, forced, delta, timestamp, timestamp)
async def _market_info(self, session, pair): """ Data from /coins/{id}. """ quote_c, base_c = pair.split('_') async with session.get( f"{self.address}coins/{quote_c}?localization=false&tickers=false&market_data=true&community_data=true&developer_data=false&sparkline=false" ) as response: data = await response.read() try: data = json.loads(data) except JSONDecodeError as jde: raise Exception( 'Rate limit possibly exceeded\nReturned error: {!s}\nReturned response content from HTTP request: {!s}' .format(jde, data)) timestamp = timestamp_normalize(self.id, data['last_updated']) if (pair not in self.last_market_info_update) or ( self.last_market_info_update[pair] < timestamp): self.last_market_info_update[pair] = timestamp # `None` and null data is systematically replaced with '-1' for digits and '' for string (empty string), for compatibility with Redis stream. market_data = { k: (-1 if (not v or (isinstance(v, dict) and not (base_c in v and v[base_c])) ) else v if k in OTHER_MARKET_DATA_FILTER else v[base_c]) for k, v in data['market_data'].items() if k in ALL_MARKET_DATA } # 'last_updated' here is assumed to be specific for market data, so it is kept as well. market_data['last_updated'] = timestamp_normalize( self.id, data['market_data']['last_updated']) community_data = { k: (v if v else -1) for k, v in data['community_data'].items() } public_interest_stats = { k: (v if v else -1) for k, v in data['public_interest_stats'].items() } # Only retain selected data, and remove as well `market_data`, `community_data` and `public_interest_stats`. # These latter are added back in `data` to have it in the shape of a flatten dict. data_s = { k: (v if v else '') for k, v in data.items() if k in MARKET_INFO_FILTER_S } data_d = { k: (v if v else -1) for k, v in data.items() if k in MARKET_INFO_FILTER_D } status = str(data['status_updates']) data = { **data_s, **data_d, **market_data, **community_data, **public_interest_stats } # `list` data type is converted to string for compatibility with Redis stream. data['status_updates'] = status await self.callback(MARKET_INFO, feed=self.id, pair=pair_exchange_to_std(pair), timestamp=timestamp, **data) return