def test_track_single_book_snapshot_message_with_past_diffs(self, mock_utils):
# Mocks binance_utils for BinanceOrderBook.diff_message_from_exchange()
mock_utils.return_value = self.trading_pair
snapshot_msg: OrderBookMessage = BinanceOrderBook.snapshot_message_from_exchange(
msg={
"trading_pair": self.trading_pair,
"lastUpdateId": 1,
"bids": [
["4.00000000", "431.00000000"]
],
"asks": [
["4.00000200", "12.00000000"]
]
},
timestamp=time.time()
)
past_diff_msg: OrderBookMessage = BinanceOrderBook.diff_message_from_exchange(
msg={
"e": "depthUpdate",
"E": 123456789,
"s": "BNBBTC",
"U": 1,
"u": 2,
"b": [
[
"0.0024",
"10"
]
],
"a": [
[
"0.0026",
"100"
]
]
}
)
self.tracking_task = self.ev_loop.create_task(
self.tracker._track_single_book(self.trading_pair)
)
self.ev_loop.run_until_complete(asyncio.sleep(0.5))
self._simulate_message_enqueue(self.tracker._past_diffs_windows[self.trading_pair], past_diff_msg)
self._simulate_message_enqueue(self.tracker._tracking_message_queues[self.trading_pair], snapshot_msg)
self.ev_loop.run_until_complete(asyncio.sleep(0.5))
self.assertEqual(1, self.tracker.order_books[self.trading_pair].snapshot_uid)
self.assertEqual(2, self.tracker.order_books[self.trading_pair].last_diff_uid)
async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
while True:
try:
ws_path: str = "/".join([
f"{convert_to_exchange_trading_pair(trading_pair).lower()}@depth"
for trading_pair in self._trading_pairs
])
url = DIFF_STREAM_URL.format(self._domain)
stream_url: str = f"{url}/{ws_path}"
async with websockets.connect(stream_url) as ws:
ws: websockets.WebSocketClientProtocol = ws
async for raw_msg in self._inner_messages(ws):
msg = ujson.loads(raw_msg)
order_book_message: OrderBookMessage = BinanceOrderBook.diff_message_from_exchange(
msg, time.time())
output.put_nowait(order_book_message)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
"Unexpected error with WebSocket connection. Retrying after 30 seconds...",
exc_info=True)
await asyncio.sleep(30.0)
def test_track_single_book_diff_message(self, mock_utils):
# Mocks binance_utils for BinanceOrderBook.diff_message_from_exchange()
mock_utils.return_value = self.trading_pair
diff_msg: OrderBookMessage = BinanceOrderBook.diff_message_from_exchange(
msg={
"e": "depthUpdate",
"E": 123456789,
"s": "BNBBTC",
"U": 1,
"u": 2,
"b": [
[
"0.0024",
"10"
]
],
"a": [
[
"0.0026",
"100"
]
]
}
)
self._simulate_message_enqueue(self.tracker._tracking_message_queues[self.trading_pair], diff_msg)
self.tracking_task = self.ev_loop.create_task(
self.tracker._track_single_book(self.trading_pair)
)
self.ev_loop.run_until_complete(asyncio.sleep(0.5))
self.assertEqual(0, self.tracker.order_books[self.trading_pair].snapshot_uid)
self.assertEqual(2, self.tracker.order_books[self.trading_pair].last_diff_uid)
async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
ws = None
while True:
try:
ws = await self._create_websocket_connection()
payload = {
"method":
"SUBSCRIBE",
"params": [
f"{binance_utils.convert_to_exchange_trading_pair(trading_pair).lower()}@depth"
for trading_pair in self._trading_pairs
],
"id":
self.DIFF_STREAM_ID
}
await ws.send_json(payload)
async for json_msg in self._iter_messages(ws):
if "result" in json_msg:
continue
order_book_message: OrderBookMessage = BinanceOrderBook.diff_message_from_exchange(
json_msg, time.time())
output.put_nowait(order_book_message)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
"Unexpected error with WebSocket connection. Retrying after 30 seconds...",
exc_info=True)
finally:
ws and await ws.close()
await self._sleep(30.0)
async def get_tracking_pairs(self) -> Dict[str, OrderBookTrackerEntry]:
auth: aiohttp.BasicAuth = aiohttp.BasicAuth(
login=conf.coinalpha_order_book_api_username,
password=conf.coinalpha_order_book_api_password)
client_session: aiohttp.ClientSession = await self.get_client_session()
response: aiohttp.ClientResponse = await client_session.get(
self.SNAPSHOT_REST_URL, auth=auth)
timestamp: float = time.time()
if response.status != 200:
raise EnvironmentError(
f"Error fetching order book tracker snapshot from {self.SNAPSHOT_REST_URL}."
)
binary_data: bytes = await response.read()
order_book_tracker_data: Dict[str, Tuple[
pd.DataFrame, pd.DataFrame]] = pickle.loads(binary_data)
retval: Dict[str, OrderBookTrackerEntry] = {}
for trading_pair, (bids_df,
asks_df) in order_book_tracker_data.items():
order_book: BinanceOrderBook = BinanceOrderBook()
order_book.apply_numpy_snapshot(bids_df.values, asks_df.values)
retval[trading_pair] = OrderBookTrackerEntry(
trading_pair, timestamp, order_book)
return retval
def test_diff_message_from_exchange(self):
diff_msg = BinanceOrderBook.diff_message_from_exchange(
msg={
"e": "depthUpdate",
"E": 123456789,
"s": "COINALPHAHBOT",
"U": 1,
"u": 2,
"b": [["0.0024", "10"]],
"a": [["0.0026", "100"]]
},
timestamp=1640000000.0,
metadata={"trading_pair": "COINALPHA-HBOT"})
self.assertEqual("COINALPHA-HBOT", diff_msg.trading_pair)
self.assertEqual(OrderBookMessageType.DIFF, diff_msg.type)
self.assertEqual(1640000000.0, diff_msg.timestamp)
self.assertEqual(2, diff_msg.update_id)
self.assertEqual(1, diff_msg.first_update_id)
self.assertEqual(-1, diff_msg.trade_id)
self.assertEqual(1, len(diff_msg.bids))
self.assertEqual(0.0024, diff_msg.bids[0].price)
self.assertEqual(10.0, diff_msg.bids[0].amount)
self.assertEqual(2, diff_msg.bids[0].update_id)
self.assertEqual(1, len(diff_msg.asks))
self.assertEqual(0.0026, diff_msg.asks[0].price)
self.assertEqual(100.0, diff_msg.asks[0].amount)
self.assertEqual(2, diff_msg.asks[0].update_id)
async def listen_for_order_book_diffs(self,
ev_loop: asyncio.AbstractEventLoop,
output: asyncio.Queue):
"""
Reads the order diffs events queue. For each event creates a diff message instance and adds it to the
output queue
:param ev_loop: the event loop the method will run in
:param output: a queue to add the created diff messages
"""
message_queue = self._message_queue[CONSTANTS.DIFF_EVENT_TYPE]
while True:
try:
json_msg = await message_queue.get()
if "result" in json_msg:
continue
trading_pair = await BinanceAPIOrderBookDataSource.trading_pair_associated_to_exchange_symbol(
symbol=json_msg["s"],
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler,
time_synchronizer=self._time_synchronizer)
order_book_message: OrderBookMessage = BinanceOrderBook.diff_message_from_exchange(
json_msg, time.time(), {"trading_pair": trading_pair})
output.put_nowait(order_book_message)
except asyncio.CancelledError:
raise
except Exception:
self.logger().exception(
"Unexpected error when processing public order book updates from exchange"
)
async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
while True:
try:
async with aiohttp.ClientSession() as client:
for trading_pair in self._trading_pairs:
try:
snapshot: Dict[str, Any] = await self.get_snapshot(client, trading_pair,
domain=self._domain)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = BinanceOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair}
)
output.put_nowait(snapshot_msg)
self.logger().debug(f"Saved order book snapshot for {trading_pair}")
# Be careful not to go above Binance's API rate limits.
await asyncio.sleep(5.0)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.", exc_info=True)
await asyncio.sleep(5.0)
this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0)
next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1)
delta: float = next_hour.timestamp() - time.time()
await asyncio.sleep(delta)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.", exc_info=True)
await asyncio.sleep(5.0)
async def _parse_order_book_diff_message(self, raw_message: Dict[str, Any],
message_queue: asyncio.Queue):
if "result" not in raw_message:
trading_pair = await self._connector.trading_pair_associated_to_exchange_symbol(
symbol=raw_message["s"])
order_book_message: OrderBookMessage = BinanceOrderBook.diff_message_from_exchange(
raw_message, time.time(), {"trading_pair": trading_pair})
message_queue.put_nowait(order_book_message)
async def _order_book_snapshot(self,
trading_pair: str) -> OrderBookMessage:
snapshot: Dict[str, Any] = await self._request_order_book_snapshot(
trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = BinanceOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair})
return snapshot_msg
def setUp(self) -> None:
super().setUp()
self.throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self.tracker: BinanceOrderBookTracker = BinanceOrderBookTracker(trading_pairs=[self.trading_pair],
throttler=self.throttler)
self.tracking_task: Optional[asyncio.Task] = None
# Simulate start()
self.tracker._order_books[self.trading_pair] = BinanceOrderBook()
self.tracker._tracking_message_queues[self.trading_pair] = asyncio.Queue()
self.tracker._past_diffs_windows[self.trading_pair] = deque()
self.tracker._order_books_initialized.set()
async def get_new_order_book(self, trading_pair: str) -> OrderBook:
async with aiohttp.ClientSession() as client:
snapshot: Dict[str, Any] = await self.get_snapshot(client, trading_pair, 1000, self._domain)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = BinanceOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair}
)
order_book = self.order_book_create_function()
order_book.apply_snapshot(snapshot_msg.bids, snapshot_msg.asks, snapshot_msg.update_id)
return order_book
async def get_new_order_book(self, trading_pair: str) -> OrderBook:
"""
Creates a local instance of the exchange order book for a particular trading pair
:param trading_pair: the trading pair for which the order book has to be retrieved
:return: a local copy of the current order book in the exchange
"""
snapshot: Dict[str, Any] = await self.get_snapshot(trading_pair, 1000)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = BinanceOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair})
order_book = self.order_book_create_function()
order_book.apply_snapshot(snapshot_msg.bids, snapshot_msg.asks,
snapshot_msg.update_id)
return order_book
def test_track_single_book_snapshot_message_no_past_diffs(self):
snapshot_msg: OrderBookMessage = BinanceOrderBook.snapshot_message_from_exchange(
msg={
"trading_pair": self.trading_pair,
"lastUpdateId": 1,
"bids": [["4.00000000", "431.00000000"]],
"asks": [["4.00000200", "12.00000000"]]
},
timestamp=time.time())
self._simulate_message_enqueue(
self.tracker._tracking_message_queues[self.trading_pair],
snapshot_msg)
self.tracking_task = self.ev_loop.create_task(
self.tracker._track_single_book(self.trading_pair))
self.ev_loop.run_until_complete(asyncio.sleep(0.5))
self.assertEqual(
1, self.tracker.order_books[self.trading_pair].snapshot_uid)
def test_snapshot_message_from_exchange(self):
snapshot_message = BinanceOrderBook.snapshot_message_from_exchange(
msg={
"lastUpdateId": 1,
"bids": [["4.00000000", "431.00000000"]],
"asks": [["4.00000200", "12.00000000"]]
},
timestamp=1640000000.0,
metadata={"trading_pair": "COINALPHA-HBOT"})
self.assertEqual("COINALPHA-HBOT", snapshot_message.trading_pair)
self.assertEqual(OrderBookMessageType.SNAPSHOT, snapshot_message.type)
self.assertEqual(1640000000.0, snapshot_message.timestamp)
self.assertEqual(1, snapshot_message.update_id)
self.assertEqual(-1, snapshot_message.trade_id)
self.assertEqual(1, len(snapshot_message.bids))
self.assertEqual(4.0, snapshot_message.bids[0].price)
self.assertEqual(431.0, snapshot_message.bids[0].amount)
self.assertEqual(1, snapshot_message.bids[0].update_id)
self.assertEqual(1, len(snapshot_message.asks))
self.assertEqual(4.000002, snapshot_message.asks[0].price)
self.assertEqual(12.0, snapshot_message.asks[0].amount)
self.assertEqual(1, snapshot_message.asks[0].update_id)
async def listen_for_order_book_snapshots(
self, ev_loop: asyncio.AbstractEventLoop, output: asyncio.Queue):
"""
This method runs continuously and request the full order book content from the exchange every hour.
The method uses the REST API from the exchange because it does not provide an endpoint to get the full order
book through websocket. With the information creates a snapshot messages that is added to the output queue
:param ev_loop: the event loop the method will run in
:param output: a queue to add the created snapshot messages
"""
while True:
try:
for trading_pair in self._trading_pairs:
try:
snapshot: Dict[str, Any] = await self.get_snapshot(
trading_pair=trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = BinanceOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair})
output.put_nowait(snapshot_msg)
self.logger().debug(
f"Saved order book snapshot for {trading_pair}")
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
f"Unexpected error fetching order book snapshot for {trading_pair}.",
exc_info=True)
await self._sleep(5.0)
await self._sleep(self.ONE_HOUR)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.", exc_info=True)
await self._sleep(5.0)
def test_trade_message_from_exchange(self):
trade_update = {
"e": "trade",
"E": 1234567890123,
"s": "COINALPHAHBOT",
"t": 12345,
"p": "0.001",
"q": "100",
"b": 88,
"a": 50,
"T": 123456785,
"m": True,
"M": True
}
trade_message = BinanceOrderBook.trade_message_from_exchange(
msg=trade_update, metadata={"trading_pair": "COINALPHA-HBOT"})
self.assertEqual("COINALPHA-HBOT", trade_message.trading_pair)
self.assertEqual(OrderBookMessageType.TRADE, trade_message.type)
self.assertEqual(1234567890.123, trade_message.timestamp)
self.assertEqual(-1, trade_message.update_id)
self.assertEqual(-1, trade_message.first_update_id)
self.assertEqual(12345, trade_message.trade_id)
async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
while True:
try:
ws_path: str = "/".join([
f"{binance_utils.convert_to_exchange_trading_pair(trading_pair).lower()}@trade"
for trading_pair in self._trading_pairs
])
url = CONSTANTS.WSS_URL.format(self._domain)
stream_url: str = f"{url}/{ws_path}"
ws = await websockets.connect(stream_url)
async for raw_msg in self._inner_messages(ws):
msg = ujson.loads(raw_msg)
trade_msg: OrderBookMessage = BinanceOrderBook.trade_message_from_exchange(
msg)
output.put_nowait(trade_msg)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
"Unexpected error with WebSocket connection. Retrying after 30 seconds...",
exc_info=True)
await asyncio.sleep(30.0)