def test_poloniex_pair_conversions():
for _, pair in poloniex_pair_mapping.items():
std = pair_exchange_to_std(pair)
assert (pair == pair_std_to_exchange(std, POLONIEX))
def get_active_symbols():
load_exchange_pair_mapping(Upbit.id)
symbols = []
for data in Upbit.get_active_symbols_info():
symbols.append(pair_exchange_to_std(data['market']))
return symbols
def test_coinbase_pair_conversions():
for _, pair in coinbase_pair_mapping.items():
assert (pair_exchange_to_std(pair) == pair_std_to_exchange(
pair, COINBASE))
async def _book(self, msg: dict, chan_id: int, timestamp: float):
delta = {BID: [], ASK: []}
msg_type = msg[0][0]
pair = None
forced = False
# initial update (i.e. snapshot)
if msg_type == 'i':
forced = True
pair = msg[0][1]['currencyPair']
pair = pair_exchange_to_std(pair)
self.l2_book[pair] = {BID: sd(), ASK: sd()}
# 0 is asks, 1 is bids
order_book = msg[0][1]['orderBook']
for key in order_book[0]:
amount = Decimal(order_book[0][key])
price = Decimal(key)
self.l2_book[pair][ASK][price] = amount
for key in order_book[1]:
amount = Decimal(order_book[1][key])
price = Decimal(key)
self.l2_book[pair][BID][price] = amount
else:
pair = self.pair_mapping[chan_id]
pair = pair_exchange_to_std(pair)
for update in msg:
msg_type = update[0]
# order book update
if msg_type == 'o':
side = ASK if update[1] == 0 else BID
price = Decimal(update[2])
amount = Decimal(update[3])
if amount == 0:
delta[side].append((price, 0))
del self.l2_book[pair][side][price]
else:
delta[side].append((price, amount))
self.l2_book[pair][side][price] = amount
elif msg_type == 't':
# index 1 is trade id, 2 is side, 3 is price, 4 is amount, 5 is timestamp
_, order_id, _, price, amount, server_ts = update
price = Decimal(price)
amount = Decimal(amount)
side = BUY if update[2] == 1 else SELL
if self.__do_callback(TRADES, pair):
await self.callback(TRADES,
feed=self.id,
pair=pair,
side=side,
amount=amount,
price=price,
timestamp=float(server_ts),
order_id=order_id,
receipt_timestamp=timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id,
msg)
if self.__do_callback(L2_BOOK, pair):
await self.book_callback(self.l2_book[pair], L2_BOOK, pair, forced,
delta, timestamp, timestamp)
async def _book_update(self, msg: dict, timestamp: float):
"""
Snapshot:
[
[
'AE',
'1',
'BTC_USDT',
'1547941504',
{
'asks':[
[
'25000.00000000',
'0.02000000'
],
[
'19745.83000000',
'0.00200000'
],
[
'19698.96000000',
'0.00100000'
],
...
]
},
{
'bids':[
[
'3662.83040000',
'0.00100000'
],
[
'3662.77540000',
'0.01000000'
],
[
'3662.59900000',
'0.10300000'
],
...
]
}
]
]
Update:
['E', '1', '1547942636', 'BTC_USDT', 'ASK', '3674.91740000', '0.02600000']
"""
forced = False
delta = {BID: [], ASK: []}
if msg[0] == 'AE':
# snapshot
forced = True
pair = pair_exchange_to_std(msg[2])
ts = msg[3]
asks = msg[4]['asks'] if 'asks' in msg[4] else msg[5]['asks']
bids = msg[5]['bids'] if 'bids' in msg[5] else msg[4]['bids']
self.l2_book[pair] = {
BID:
sd({Decimal(price): Decimal(amount)
for price, amount in bids}),
ASK:
sd({Decimal(price): Decimal(amount)
for price, amount in asks})
}
else:
# Update
ts = msg[2]
pair = pair_exchange_to_std(msg[3])
side = ASK if msg[4] == 'ASK' else BID
price = Decimal(msg[5])
amount = Decimal(msg[6])
if amount == 0:
if price in self.l2_book[pair][side]:
del self.l2_book[pair][side][price]
delta[side].append((price, 0))
else:
self.l2_book[pair][side][price] = amount
delta[side].append((price, amount))
await self.book_callback(self.l2_book[pair], L2_BOOK, pair, forced,
delta, ts, timestamp)
def test_gemini_pair_conversions():
for _, pair in gemini_pair_mapping.items():
std = pair_exchange_to_std(pair)
assert (pair == pair_std_to_exchange(std, GEMINI))
async def _raw_book(self, msg):
chan_id = msg[0]
pair = self.channel_map[chan_id]['symbol']
pair = pair_exchange_to_std(pair)
if isinstance(msg[1], list):
if isinstance(msg[1][0], list):
# snapshot so clear book
self.l2_book[pair] = {BID: sd(), ASK: sd()}
for update in msg[1]:
order_id, price, amount = update
price = Decimal(price)
amount = Decimal(amount)
if amount > 0:
side = BID
else:
side = ASK
amount = abs(amount)
if price not in self.l2_book[pair][side]:
self.l2_book[pair][side][price] = amount
self.order_map[order_id] = {
'price': price,
'amount': amount,
'side': side
}
else:
self.l2_book[pair][side][price]
self.l2_book[pair][side][price] += amount
self.order_map[order_id] = {
'price': price,
'amount': amount,
'side': side
}
else:
# book update
order_id, price, amount = [Decimal(x) for x in msg[1]]
if amount > 0:
side = BID
else:
side = ASK
amount = abs(amount)
if price == 0:
price = self.order_map[order_id]['price']
self.l2_book[pair][side][price] -= self.order_map[
order_id]['amount']
if self.l2_book[pair][side][price] == 0:
del self.l2_book[pair][side][price]
del self.order_map[order_id]
else:
self.order_map[order_id] = {
'price': price,
'amount': amount,
'side': side
}
if price in self.l2_book[pair][side]:
self.l2_book[pair][side][price]
self.l2_book[pair][side][price] += amount
else:
self.l2_book[pair][side][price] = amount
elif msg[1] == 'hb':
pass
else:
LOG.warning("{} - Unexpected book msg {}".format(self.id, msg))
if L3_BOOK in self.standardized_channels:
await self.callbacks[L3_BOOK](feed=self.id,
pair=pair,
book=self.l2_book[pair])
else:
await self.callbacks[L2_BOOK](feed=self.id,
pair=pair,
book=self.l2_book[pair])
async def _book(self, msg, chan_id, sequence):
msg_type = msg[0][0]
pair = None
# initial update (i.e. snapshot)
if msg_type == 'i':
pair = msg[0][1]['currencyPair']
pair = pair_exchange_to_std(pair)
self.l3_book[pair] = {BID: sd(), ASK: sd()}
# 0 is asks, 1 is bids
order_book = msg[0][1]['orderBook']
for key in order_book[0]:
amount = Decimal(order_book[0][key])
price = Decimal(key)
self.l3_book[pair][ASK][price] = amount
for key in order_book[1]:
amount = Decimal(order_book[1][key])
price = Decimal(key)
self.l3_book[pair][BID][price] = amount
else:
pair = poloniex_id_pair_mapping[chan_id]
pair = pair_exchange_to_std(pair)
for update in msg:
timestamp = None
msg_type = update[0]
# order book update
if msg_type == 'o':
mtype = 'change'
side = ASK if update[1] == 0 else BID
price = Decimal(update[2])
amount = Decimal(update[3])
if amount == 0:
del self.l3_book[pair][side][price]
else:
self.l3_book[pair][side][price] = amount
elif msg_type == 't':
# index 1 is trade id, 2 is side, 3 is price, 4 is amount, 5 is timestamp
mtype = 'trade'
timestamp = self.tz_aware_datetime_from_string(update[5])
price = Decimal(update[3])
side = ASK if update[2] == 0 else BID
amount = Decimal(update[4])
await self.callbacks[TRADES](feed=self.id,
pair=pair,
side=side,
amount=amount,
price=price)
else:
LOG.warning("{} - Unexpected message received: {}".format(self.id, msg))
# continue so we don't hit the callback below if the msg_type isn't of the 2 tested for above
continue
await self.callbacks[L3_BOOK_UPDATE](feed=self.id,
pair=pair,
msg_type=mtype,
timestamp=timestamp,
sequence=sequence,
side=side,
price=price,
size=amount)
await self.callbacks[L3_BOOK](feed=self.id,
sequence=sequence,
timestamp=None,
pair=pair,
book=self.l3_book[pair])
def test_dsx_pair_conversion():
load_exchange_pair_mapping(DSX)
for _, pair in dsx_pairs().items():
std = pair_exchange_to_std(pair)
assert (pair == pair_std_to_exchange(std, DSX))
async def _book_update(self, msg):
"""
Snapshot:
[
[
'AE',
'1',
'BTC_USDT',
'1547941504',
{
'asks':[
[
'25000.00000000',
'0.02000000'
],
[
'19745.83000000',
'0.00200000'
],
[
'19698.96000000',
'0.00100000'
],
...
]
},
{
'bids':[
[
'3662.83040000',
'0.00100000'
],
[
'3662.77540000',
'0.01000000'
],
[
'3662.59900000',
'0.10300000'
],
...
]
}
]
]
Update:
['E', '1', '1547942636', 'BTC_USDT', 'ASK', '3674.91740000', '0.02600000']
"""
if msg[0] == 'AE':
# snapshot
pair = pair_exchange_to_std(msg[2])
timestamp = msg[3]
asks = msg[4]['asks'] if 'asks' in msg[4] else msg[5]['asks']
bids = msg[5]['bids'] if 'bids' in msg[5] else msg[4]['bids']
self.l2_book[pair] = {
BID: sd({
Decimal(price): Decimal(amount)
for price, amount in bids
}),
ASK: sd({
Decimal(price): Decimal(amount)
for price, amount in asks
})
}
else:
# Update
timestamp = msg[2]
pair = pair_exchange_to_std(msg[3])
side = ASK if msg[4] == 'ASK' else BID
price = Decimal(msg[5])
amount = Decimal(msg[6])
if amount == 0:
try:
del self.l2_book[pair][side][price]
except BaseException:
pass
else:
self.l2_book[pair][side][price] = amount
await self.book_callback(pair, L2_BOOK, True, None, timestamp)
async def _ticker(self, msg):
await self.callbacks[TICKER](feed=self.id,
pair=pair_exchange_to_std(msg['symbol']),
bid=Decimal(msg['bid']),
ask=Decimal(msg['ask']))
async def ticker(self, msg: dict, timestamp: float):
for t in msg['D']:
if (not self.config and t['M'] in self.pairs) or ('SubscribeToSummaryDeltas' in self.config and t['M'] in self.config['SubscribeToSummaryDeltas']):
await self.callback(TICKER, feed=self.id, pair=pair_exchange_to_std(t['M']), bid=Decimal(t['B']), ask=Decimal(t['A']), timestamp=timestamp_normalize(self.id, t['T']), receipt_timestamp=timestamp)
def test_blockchain_pair_conversions():
load_exchange_pair_mapping(BLOCKCHAIN)
for _, pair in blockchain_pairs().items():
assert (pair_exchange_to_std(pair) == pair_std_to_exchange(
pair, BLOCKCHAIN))
async def _ticker(self, msg):
await self.callback(TICKER, feed=self.id,
pair=pair_exchange_to_std(msg['symbol']),
bid=Decimal(msg['bid']),
ask=Decimal(msg['ask']),
timestamp=timestamp_normalize(self.id, msg['timestamp']))
def test_bitfinex_pair_conversions():
for _, pair in bitfinex_pair_mapping.items():
std = pair_exchange_to_std(pair)
assert (pair == pair_std_to_exchange(std, BITFINEX))
def test_bitcoincom_pair_conversions():
load_exchange_pair_mapping(BITCOINCOM)
for _, pair in bitcoincom_pairs().items():
std = pair_exchange_to_std(pair)
assert (pair == pair_std_to_exchange(std, BITCOINCOM))
def test_hitbtc_pair_conversions():
for _, pair in hitbtc_pair_mapping.items():
std = pair_exchange_to_std(pair)
assert (pair == pair_std_to_exchange(std, HITBTC))
async def _l2_update(self, msg: dict, timestamp: float):
'''
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok",
"lag":0,
"order_books":[
{
"side":"buy",
"price":"0.0165",
"quantity":"0.47"
},{
"side":"buy",
"price":"0",
"quantity":"14656.177"
},{
"side":"sell",
"price":"6400",
"quantity":"0.001"
}],
"reset":true
}
{
"channel":"marketdata",
"market_id":"ETH-BTC",
"status":"ok",
"lag":0,
"order_books":[
{
"side":"buy",
"price":"0.0281",
"quantity":"48.541"
},{
"side":"sell",
"price":"0.0283",
"quantity":"0"
}]
}
'''
pair = pair_exchange_to_std(msg['market_id'])
is_snapshot = msg.get('reset', False)
if is_snapshot:
self.l2_book[pair] = {ASK: sd(), BID: sd()}
for entry in msg["order_books"]:
price = Decimal(entry['price'])
quantity = Decimal(entry['quantity'])
side = BID if entry['side'] == "buy" else ASK
self.l2_book[pair][side][price] = quantity
await self.book_callback(self.l2_book[pair], L2_BOOK, pair, True, None, timestamp, timestamp)
else:
delta = {BID: [], ASK: []}
for entry in msg["order_books"]:
price = Decimal(entry['price'])
quantity = Decimal(entry['quantity'])
side = BID if entry['side'] == "buy" else ASK
if quantity == 0:
if price in self.l2_book[pair][side]:
del self.l2_book[pair][side][price]
delta[side].append((price, 0))
else:
self.l2_book[pair][side][price] = quantity
delta[side].append((price, quantity))
await self.book_callback(self.l2_book[pair], L2_BOOK, pair, False, delta, timestamp, timestamp)
def test_bitstamp_pair_conversions():
for _, pair in bitstamp_pair_mapping.items():
std = pair_exchange_to_std(pair)
assert (pair == pair_std_to_exchange(std, BITSTAMP))
def __reset(self, pairs):
for pair in pairs:
self.l2_book[pair_exchange_to_std(pair)] = {BID: sd(), ASK: sd()}
async def _raw_book(self, msg: dict, timestamp: float):
"""
For L3 book updates
"""
def add_to_book(pair, side, price, order_id, amount):
if price in self.l3_book[pair][side]:
self.l3_book[pair][side][price][order_id] = amount
else:
self.l3_book[pair][side][price] = {order_id: amount}
def remove_from_book(pair, side, order_id):
price = self.order_map[pair][side][order_id]['price']
del self.l3_book[pair][side][price][order_id]
if len(self.l3_book[pair][side][price]) == 0:
del self.l3_book[pair][side][price]
delta = {BID: [], ASK: []}
forced = False
chan_id = msg[0]
pair = self.channel_map[chan_id]['symbol']
pair = pair_exchange_to_std(pair)
if isinstance(msg[1], list):
if isinstance(msg[1][0], list):
# snapshot so clear orders
self.order_map[pair] = {BID: {}, ASK: {}}
self.l3_book[pair] = {BID: sd(), ASK: sd()}
for update in msg[1]:
order_id, price, amount = update
price = Decimal(price)
amount = Decimal(amount)
if amount > 0:
side = BID
else:
side = ASK
amount = abs(amount)
self.order_map[pair][side][order_id] = {
'price': price,
'amount': amount
}
add_to_book(pair, side, price, order_id, amount)
forced = True
else:
# book update
order_id, price, amount = msg[1]
price = Decimal(price)
amount = Decimal(amount)
if amount > 0:
side = BID
else:
side = ASK
amount = abs(amount)
if price == 0:
price = self.order_map[pair][side][order_id]['price']
remove_from_book(pair, side, order_id)
del self.order_map[pair][side][order_id]
delta[side].append((order_id, price, 0))
else:
if order_id in self.order_map[pair][side]:
del_price = self.order_map[pair][side][order_id][
'price']
delta[side].append((order_id, del_price, 0))
# remove existing order before adding new one
delta[side].append((order_id, price, amount))
remove_from_book(pair, side, order_id)
else:
delta[side].append((order_id, price, amount))
add_to_book(pair, side, price, order_id, amount)
self.order_map[pair][side][order_id] = {
'price': price,
'amount': amount
}
elif msg[1] == 'hb':
return
else:
LOG.warning("%s: Unexpected book msg %s", self.id, msg)
return
await self.book_callback(self.l3_book[pair], L3_BOOK, pair, forced,
delta, timestamp, timestamp)
async def _market_info(self, session, pair):
"""
Data from /coins/{id}.
"""
quote_c, base_c = pair.split('_')
async with session.get(
f"{self.address}coins/{quote_c}?localization=false&tickers=false&market_data=true&community_data=true&developer_data=false&sparkline=false"
) as response:
data = await response.read()
try:
data = json.loads(data)
except JSONDecodeError as jde:
raise Exception(
'Rate limit possibly exceeded\nReturned error: {!s}\nReturned response content from HTTP request: {!s}'
.format(jde, data))
timestamp = timestamp_normalize(self.id, data['last_updated'])
if (pair not in self.last_market_info_update) or (
self.last_market_info_update[pair] < timestamp):
self.last_market_info_update[pair] = timestamp
# `None` and null data is systematically replaced with '-1' for digits and '' for string (empty string), for compatibility with Redis stream.
market_data = {
k:
(-1 if
(not v or
(isinstance(v, dict) and not (base_c in v and v[base_c]))
) else v if k in OTHER_MARKET_DATA_FILTER else v[base_c])
for k, v in data['market_data'].items()
if k in ALL_MARKET_DATA
}
# 'last_updated' here is assumed to be specific for market data, so it is kept as well.
market_data['last_updated'] = timestamp_normalize(
self.id, data['market_data']['last_updated'])
community_data = {
k: (v if v else -1)
for k, v in data['community_data'].items()
}
public_interest_stats = {
k: (v if v else -1)
for k, v in data['public_interest_stats'].items()
}
# Only retain selected data, and remove as well `market_data`, `community_data` and `public_interest_stats`.
# These latter are added back in `data` to have it in the shape of a flatten dict.
data_s = {
k: (v if v else '')
for k, v in data.items() if k in MARKET_INFO_FILTER_S
}
data_d = {
k: (v if v else -1)
for k, v in data.items() if k in MARKET_INFO_FILTER_D
}
status = str(data['status_updates'])
data = {
**data_s,
**data_d,
**market_data,
**community_data,
**public_interest_stats
}
# `list` data type is converted to string for compatibility with Redis stream.
data['status_updates'] = status
await self.callback(MARKET_INFO,
feed=self.id,
pair=pair_exchange_to_std(pair),
timestamp=timestamp,
**data)
return
