async def _collect_and_decode_messages_loop(self, stream_type: StreamType,
task_index: int,
output: asyncio.Queue):
while True:
try:
kucoin_msg_iterator: KucoinWSConnectionIterator = KucoinWSConnectionIterator(
stream_type,
self._tasks[stream_type][task_index].trading_pairs,
self._throttler)
self._tasks[stream_type][
task_index].message_iterator = kucoin_msg_iterator
async for raw_msg in kucoin_msg_iterator:
msg_type: str = raw_msg.get("type", "")
if msg_type in {"ack", "welcome", "pong"}:
pass
elif msg_type == "message":
if stream_type == StreamType.Depth:
order_book_message: OrderBookMessage = KucoinOrderBook.diff_message_from_exchange(
raw_msg)
else:
trading_pair: str = convert_to_exchange_trading_pair(
raw_msg["data"]["symbol"])
data = raw_msg["data"]
order_book_message: OrderBookMessage = \
KucoinOrderBook.trade_message_from_exchange(
data,
metadata={"trading_pair": trading_pair}
)
output.put_nowait(order_book_message)
elif msg_type == "error":
self.logger().error(
f"WS error message from Kucoin: {raw_msg}")
else:
self.logger().warning(
f"Unrecognized message type from Kucoin: {msg_type}. "
f"Message = {raw_msg}.")
except asyncio.CancelledError:
raise
except asyncio.TimeoutError:
self.logger().error(
"Timeout error with WebSocket connection. Retrying after 5 seconds...",
exc_info=True)
await asyncio.sleep(5.0)
except Exception:
self.logger().error(
"Unexpected exception with WebSocket connection. Retrying after 5 seconds...",
exc_info=True)
await asyncio.sleep(5.0)
finally:
if stream_type in self._tasks:
if task_index in self._tasks:
self._tasks[stream_type][
task_index].message_iterator = None
async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
while True:
try:
trading_pairs: List[str] = self._trading_pairs if self._trading_pairs else await self.fetch_trading_pairs()
async with aiohttp.ClientSession() as client:
for trading_pair in trading_pairs:
try:
snapshot: Dict[str, Any] = await self.get_snapshot(client, trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = KucoinOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"symbol": trading_pair}
)
output.put_nowait(snapshot_msg)
self.logger().debug(f"Saved order book snapshot for {trading_pair}")
await asyncio.sleep(self.SLEEP_BETWEEN_SNAPSHOT_REQUEST)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.", exc_info=True)
await asyncio.sleep(5.0)
await asyncio.sleep(secs_until_next_oclock())
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.", exc_info=True)
await asyncio.sleep(5.0)
async def listen_for_order_book_diffs(self,
ev_loop: asyncio.AbstractEventLoop,
output: asyncio.Queue):
"""
Reads the order diffs events queue. For each event creates a diff message instance and adds it to the
output queue
:param ev_loop: the event loop the method will run in
:param output: a queue to add the created diff messages
"""
message_queue = self._message_queue[CONSTANTS.DIFF_EVENT_TYPE]
while True:
try:
json_msg = await message_queue.get()
trading_pair = await self.trading_pair_associated_to_exchange_symbol(
symbol=json_msg["data"]["symbol"],
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler)
order_book_message: OrderBookMessage = KucoinOrderBook.diff_message_from_exchange(
json_msg, self._time(), {"trading_pair": trading_pair})
output.put_nowait(order_book_message)
except asyncio.CancelledError:
raise
except Exception:
self.logger().exception(
"Unexpected error when processing public order book updates from exchange"
)
async def get_new_order_book(self, trading_pair: str) -> OrderBook:
async with aiohttp.ClientSession() as client:
snapshot: Dict[str, Any] = await self.get_snapshot(client, trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = KucoinOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"symbol": trading_pair}
)
order_book: OrderBook = self.order_book_create_function()
active_order_tracker: KucoinActiveOrderTracker = KucoinActiveOrderTracker()
bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg)
order_book.apply_snapshot(bids, asks, snapshot_msg.update_id)
return order_book
async def get_new_order_book(self, trading_pair: str) -> OrderBook:
"""
Creates a local instance of the exchange order book for a particular trading pair
:param trading_pair: the trading pair for which the order book has to be retrieved
:return: a local copy of the current order book in the exchange
"""
snapshot: Dict[str, Any] = await self.get_snapshot(trading_pair)
snapshot_timestamp: float = snapshot["data"]["time"] * 1e-3
snapshot_msg: OrderBookMessage = KucoinOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair})
order_book: OrderBook = self.order_book_create_function()
order_book.apply_snapshot(snapshot_msg.bids, snapshot_msg.asks,
snapshot_msg.update_id)
return order_book
async def _take_full_order_book_snapshot(self, trading_pairs: List[str],
snapshot_queue: asyncio.Queue):
for trading_pair in trading_pairs:
try:
snapshot: Dict[str, Any] = await self.get_snapshot(
trading_pair=trading_pair)
snapshot_timestamp = float(snapshot["data"]["time"]) * 1e-3
snapshot_msg: OrderBookMessage = KucoinOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"trading_pair": trading_pair})
snapshot_queue.put_nowait(snapshot_msg)
self.logger().debug(
f"Saved order book snapshot for {trading_pair}")
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
f"Unexpected error fetching order book snapshot for {trading_pair}.",
exc_info=True)
async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
websocket_data: Dict[str, Any] = await self.ws_connect_data()
kucoin_ws_uri: str = websocket_data["data"]["instanceServers"][0][
"endpoint"] + "?token=" + websocket_data["data"][
"token"] + "&acceptUserMessage=true"
while True:
try:
async with websockets.connect(kucoin_ws_uri) as ws:
ws: websockets.WebSocketClientProtocol = ws
for trading_pair in self._trading_pairs:
subscribe_request: Dict[str, Any] = {
"id": int(time.time()),
"type": "subscribe",
"topic": f"/market/match:{trading_pair}",
"privateChannel": False,
"response": True
}
await ws.send(json.dumps(subscribe_request))
async for raw_msg in self._inner_messages(ws):
msg: Dict[str, Any] = json.loads(raw_msg)
if msg["type"] == "pong" or msg["type"] == "ack":
pass
elif msg["type"] == "message":
trading_pair = msg["data"]["symbol"]
data = msg["data"]
trade_message: OrderBookMessage = KucoinOrderBook.trade_message_from_exchange(
data, metadata={"trading_pair": trading_pair})
output.put_nowait(trade_message)
else:
self.logger().debug(
f"Unrecognized message received from Kucoin websocket: {msg}"
)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
"Unexpected error with WebSocket connection. Retrying after 30 seconds...",
exc_info=True)
await asyncio.sleep(30.0)
async def listen_for_order_book_snapshots(self,
ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
while True:
try:
async with aiohttp.ClientSession() as client:
for trading_pair in self._trading_pairs:
try:
snapshot: Dict[str, Any] = await self.get_snapshot(
client, trading_pair)
snapshot_timestamp: float = time.time()
snapshot_msg: OrderBookMessage = KucoinOrderBook.snapshot_message_from_exchange(
snapshot,
snapshot_timestamp,
metadata={"symbol": trading_pair})
output.put_nowait(snapshot_msg)
self.logger().debug(
f"Saved order book snapshot for {trading_pair}"
)
await asyncio.sleep(5.0)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.",
exc_info=True)
await asyncio.sleep(5.0)
this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(
minute=0, second=0, microsecond=0)
next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1)
delta: float = next_hour.timestamp() - time.time()
await asyncio.sleep(delta)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unexpected error.", exc_info=True)
await asyncio.sleep(5.0)