async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
"""
*required
Subscribe to diff channel via web socket, and keep the connection open for incoming messages
:param ev_loop: ev_loop to execute this function in
:param output: an async queue where the incoming messages are stored
"""
while True:
try:
trading_pairs: List[str] = await self.get_trading_pairs()
async with websockets.connect(COINBASE_WS_FEED) as ws:
ws: websockets.WebSocketClientProtocol = ws
subscribe_request: Dict[str, Any] = {
"type": "subscribe",
"product_ids": trading_pairs,
"channels": ["full"]
}
await ws.send(ujson.dumps(subscribe_request))
async for raw_msg in self._inner_messages(ws):
msg = ujson.loads(raw_msg)
msg_type: str = msg.get("type", None)
if msg_type is None:
raise ValueError(
f"Coinbase Pro Websocket message does not contain a type - {msg}"
)
elif msg_type == "error":
raise ValueError(
f"Coinbase Pro Websocket received error message - {msg['message']}"
)
elif msg_type in ["open", "match", "change", "done"]:
if msg_type == "done" and "price" not in msg:
# done messages with no price are completed market orders which can be ignored
continue
order_book_message: OrderBookMessage = CoinbaseProOrderBook.diff_message_from_exchange(
msg)
output.put_nowait(order_book_message)
elif msg_type in [
"received", "activate", "subscriptions"
]:
# these messages are not needed to track the order book
continue
else:
raise ValueError(
f"Unrecognized Coinbase Pro Websocket message received - {msg}"
)
except asyncio.CancelledError:
raise
except Exception:
self.logger().network(
f"Unexpected error with WebSocket connection.",
exc_info=True,
app_warning_msg=
f"Unexpected error with WebSocket connection. Retrying in 30 seconds. "
f"Check network connection.")
await asyncio.sleep(30.0)
async def get_new_order_book(self, trading_pair: str) -> OrderBook:
async with aiohttp.ClientSession() as client:
snapshot: Dict[str, any] = await self.get_snapshot(client, trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = CoinbaseProOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair}
)
active_order_tracker: CoinbaseProActiveOrderTracker = CoinbaseProActiveOrderTracker()
bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg)
order_book = self.order_book_create_function()
order_book.apply_snapshot(bids, asks, snapshot_msg.update_id)
return order_book
async def get_tracking_pairs(self) -> Dict[str, OrderBookTrackerEntry]:
"""
*required
Initializes order books and order book trackers for the list of trading pairs
returned by `self.get_trading_pairs`
:returns: A dictionary of order book trackers for each trading pair
"""
# Get the currently active markets
async with aiohttp.ClientSession() as client:
trading_pairs: List[str] = await self.get_trading_pairs()
retval: Dict[str, OrderBookTrackerEntry] = {}
number_of_pairs: int = len(trading_pairs)
for index, trading_pair in enumerate(trading_pairs):
try:
snapshot: Dict[str, any] = await self.get_snapshot(
client, trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = CoinbaseProOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"symbol": trading_pair})
order_book: OrderBook = self.order_book_create_function()
active_order_tracker: CoinbaseProActiveOrderTracker = CoinbaseProActiveOrderTracker(
)
bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(
snapshot_msg)
order_book.apply_snapshot(bids, asks,
snapshot_msg.update_id)
retval[trading_pair] = CoinbaseProOrderBookTrackerEntry(
trading_pair, snapshot_timestamp, order_book,
active_order_tracker)
self.logger().info(
f"Initialized order book for {trading_pair}. "
f"{index+1}/{number_of_pairs} completed.")
await asyncio.sleep(0.6)
except IOError:
self.logger().network(
f"Error getting snapshot for {trading_pair}.",
exc_info=True,
app_warning_msg=
f"Error getting snapshot for {trading_pair}. Check network connection."
)
except Exception:
self.logger().error(
f"Error initializing order book for {trading_pair}. ",
exc_info=True)
return retval
async def listen_for_order_book_snapshots(self,
ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
"""
*required
Fetches order book snapshots for each trading pair, and use them to update the local order book
:param ev_loop: ev_loop to execute this function in
:param output: an async queue where the incoming messages are stored
"""
while True:
try:
trading_pairs: List[str] = await self.get_trading_pairs()
async with aiohttp.ClientSession() as client:
for trading_pair in trading_pairs:
try:
snapshot: Dict[str, any] = await self.get_snapshot(
client, trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = CoinbaseProOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"product_id": trading_pair})
output.put_nowait(snapshot_msg)
self.logger().debug(
f"Saved order book snapshot for {trading_pair}"
)
# Be careful not to go above API rate limits.
await asyncio.sleep(5.0)
except asyncio.CancelledError:
raise
except Exception:
self.logger().network(
f"Unexpected error with WebSocket connection.",
exc_info=True,
app_warning_msg=
f"Unexpected error with WebSocket connection. Retrying in 5 seconds. "
f"Check network connection.")
await asyncio.sleep(5.0)
this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(
minute=0, second=0, microsecond=0)
next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1)
delta: float = next_hour.timestamp() - time.time()
await asyncio.sleep(delta)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.", exc_info=True)
await asyncio.sleep(5.0)
async def get_tracking_pairs(self) -> Dict[str, OrderBookTrackerEntry]:
# Get the currently active markets
async with aiohttp.ClientSession() as client:
trading_pairs: List[str] = await self.get_trading_pairs()
retval: Dict[str, OrderBookTrackerEntry] = {}
number_of_pairs: int = len(trading_pairs)
for index, trading_pair in enumerate(trading_pairs):
try:
snapshot: Dict[str, any] = await self.get_snapshot(
client, trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = self.order_book_class.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"symbol": trading_pair})
order_book: CoinbaseProOrderBook = CoinbaseProOrderBook()
active_order_tracker: CoinbaseProActiveOrderTracker = CoinbaseProActiveOrderTracker(
)
bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(
snapshot_msg)
order_book.apply_snapshot(bids, asks,
snapshot_msg.update_id)
retval[trading_pair] = CoinbaseProOrderBookTrackerEntry(
trading_pair, snapshot_timestamp, order_book,
active_order_tracker)
self.logger().info(
f"Initialized order book for {trading_pair}. "
f"{index+1}/{number_of_pairs} completed.")
await asyncio.sleep(0.6)
except Exception:
self.logger().error(
f"Error getting snapshot for {trading_pair}. ",
exc_info=True)
return retval
def test_diff_msg_get_added_to_order_book(self):
test_active_order_tracker = self.order_book_tracker._active_order_trackers[
"BTC-USD"]
price = "200"
order_id = "test_order_id"
product_id = "BTC-USD"
remaining_size = "1.00"
# Test open message diff
raw_open_message = {
"type": "open",
"time": datetime.now().isoformat(),
"product_id": product_id,
"sequence": 20000000000,
"order_id": order_id,
"price": price,
"remaining_size": remaining_size,
"side": "buy"
}
open_message = CoinbaseProOrderBook.diff_message_from_exchange(
raw_open_message)
self.order_book_tracker._order_book_diff_stream.put_nowait(
open_message)
self.run_parallel(asyncio.sleep(5))
test_order_book_row = test_active_order_tracker.active_bids[Decimal(
price)]
self.assertEqual(test_order_book_row[order_id]["remaining_size"],
remaining_size)
# Test change message diff
new_size = "2.00"
raw_change_message = {
"type": "change",
"time": datetime.now().isoformat(),
"product_id": product_id,
"sequence": 20000000001,
"order_id": order_id,
"price": price,
"new_size": new_size,
"old_size": remaining_size,
"side": "buy",
}
change_message = CoinbaseProOrderBook.diff_message_from_exchange(
raw_change_message)
self.order_book_tracker._order_book_diff_stream.put_nowait(
change_message)
self.run_parallel(asyncio.sleep(5))
test_order_book_row = test_active_order_tracker.active_bids[Decimal(
price)]
self.assertEqual(test_order_book_row[order_id]["remaining_size"],
new_size)
# Test match message diff
match_size = "0.50"
raw_match_message = {
"type": "match",
"trade_id": 10,
"sequence": 20000000002,
"maker_order_id": order_id,
"taker_order_id": "test_order_id_2",
"time": datetime.now().isoformat(),
"product_id": "BTC-USD",
"size": match_size,
"price": price,
"side": "buy"
}
match_message = CoinbaseProOrderBook.diff_message_from_exchange(
raw_match_message)
self.order_book_tracker._order_book_diff_stream.put_nowait(
match_message)
self.run_parallel(asyncio.sleep(5))
test_order_book_row = test_active_order_tracker.active_bids[Decimal(
price)]
self.assertEqual(
Decimal(test_order_book_row[order_id]["remaining_size"]),
Decimal(new_size) - Decimal(match_size))
# Test done message diff
raw_done_message = {
"type": "done",
"time": datetime.now().isoformat(),
"product_id": "BTC-USD",
"sequence": 20000000003,
"price": price,
"order_id": order_id,
"reason": "filled",
"remaining_size": 0,
"side": "buy",
}
done_message = CoinbaseProOrderBook.diff_message_from_exchange(
raw_done_message)
self.order_book_tracker._order_book_diff_stream.put_nowait(
done_message)
self.run_parallel(asyncio.sleep(5))
test_order_book_row = test_active_order_tracker.active_bids[Decimal(
price)]
self.assertTrue(order_id not in test_order_book_row)