async def process_message(msg):
line = to_native(msg)
logger.info("Got msg", extra={"DATA": line})
# if line.get("marketHours") != "REGULAR_MARKET":
# return
# {"id": "AAPL", "price": 316.70001220703125, "time": "2020-05-20T11:54:17",
# "exchange": "NMS", "quoteType": "EQUITY", "marketHours": "PRE_MARKET",
# "changePercent": 1.1368708610534668, "change": 3.55999755859375, "priceHint": "4"}
# {'id': '^RUT', 'price': 1367.7601318359375,
# 'time': datetime.datetime(2020, 6, 11, 18, 50), 'exchange': 'WCB',
# 'quoteType': 'INDEX', 'marketHours': 'REGULAR_MARKET',
# 'changePercent': -6.789621353149414, 'change': -99.6302490234375,
# 'priceHint': '4'}
try:
upd_kwargs = dict(
symbol=line["id"],
time=line["time"],
price=line["price"],
volume=line.get("dayVolume", 0),
)
except KeyError as e:
logger.exception(e)
else:
await update_minute_price(**upd_kwargs)
def unsubscribe(self, symbol, socket):
if symbol not in self._symbols:
return logger.critical(f"Symbol {symbol} not in self._symbols")
subscribers = self._symbols[symbol]
subscribers.pop(socket)
if not subscribers:
logger.info(
f"Empty subscribers set for {symbol}: discarding its cached object"
)
del self._symbols[symbol]
async def main(symbols):
if not symbols:
symbols = [s["id"] for s in await get_symbols()]
conn = aiohttp.TCPConnector(ttl_dns_cache=300)
async with aiohttp.ClientSession(connector=conn,
headers=WS_SESSION_HEADER) as session:
ws = await session.ws_connect(WS_ENDPOINT)
msg_data = {"subscribe": symbols}
await ws.send_str(json.dumps(msg_data))
logger.info("Sent sub", extra={"MESSAGE": msg_data})
while RUN:
msg = await ws.receive()
if msg.type == aiohttp.WSMsgType.TEXT:
await process_message(msg.data)
else:
logger.error(f"Unexpected msg.type {msg}")
break
async def ws_handler(request):
ws = web.WebSocketResponse()
await ws.prepare(request)
logger.info('Feed client connected. Waiting for symbols to subscribe')
queue = asyncio.Queue(maxsize=20)
functions = [] # functions to process data before sending to the recipient
loop = asyncio.get_event_loop()
t = loop.create_task(incoming_listener(ws, queue, functions))
logger.info(f'Receive loop launched {t}')
# get messages from the queue and send to client
try:
while not ws.closed:
msg = await queue.get()
# append function results
for f in functions:
msg[f] = data_processor.get_result(msg.get("symbol"), f)
# send msg
await ws.send_json(msg, dumps=json_dumps)
except ConnectionResetError as e:
logger.warning(f"ConnectionResetError at send updates {e}")
logger.info('Feed client disconnected')
return ws
async def _process_changes_loop(self):
logger.info(f"Start waiting for changes..")
async for change in watch_changes_minute_stats():
symbol = change.pop("symbol")
data = [{
k: v
for k, v in change.items()
if k in ("open", "low", "high", "close", "volume", "time")
}]
logger.info(f"Capture change of {symbol}")
data_processor.calculate_functions(symbol, data)
message = {"symbol": symbol, "data": data}
if symbol in self._symbols:
subscribers = self._symbols[symbol]
dead_sockets = []
for socket, subscriber in subscribers.items():
if subscriber.full():
dead_sockets.append(socket)
else:
subscriber.put_nowait(message)
for socket in dead_sockets:
await socket.close()
logger.info(
'Force close of socket that is not reading data')
subscribers.pop(socket)
async def watch_changes_minute_stats():
collection = get_mongodb_collection(collection_name=MONGODB_MINUTE_COLLECTION)
resume_after = None
while True:
logger.info(f"Start watching mongodb changes after {resume_after}")
changes = collection.watch(full_document="updateLookup", resume_after=resume_after)
while True:
try:
change = await changes.next()
except PyMongoError as e:
logger.error(f"Got feed error {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"})
await asyncio.sleep(MONGODB_ERROR_INTERVAL)
except StopAsyncIteration as e:
logger.exception(e)
resume_after = None
break
except Exception as e:
logger.exception(e)
await asyncio.sleep(MONGODB_ERROR_INTERVAL)
else:
resume_after = change["_id"]
if "fullDocument" in change: # present in insert/update changes
yield change["fullDocument"]
async def incoming_listener(ws, queue, functions):
subscribed_symbols = set()
try:
while not ws.closed:
incoming = await ws.receive()
logger.info("Got incoming msg", extra={"MESSAGE": incoming})
if incoming.type is WSMsgType.TEXT:
try:
data = json.loads(incoming.data)
except ValueError as e:
logger.error(e)
else:
new_functions = data.get("functions")
if new_functions:
# the same obj, but new contents
functions[:] = new_functions
subscribe = data.get("subscribe")
if subscribe:
subscribe = set(subscribe)
delete_symbols = subscribed_symbols - subscribe
for symbol in delete_symbols:
symbol_changes.unsubscribe(symbol, ws)
add_symbols = subscribe - subscribed_symbols
for symbol in add_symbols:
symbol_changes.subscribe(symbol, ws, queue)
logger.info(
f"Subscribed to {add_symbols} and unsubscribed from {delete_symbols}"
)
for symbol in add_symbols:
start_data = await get_minute_stats(symbol)
queue.put_nowait(start_data)
except ConnectionResetError as e:
logger.warning(f"ConnectionResetError while getting data {e}")
finally:
logger.info(f"Unsubscribe socket from {subscribed_symbols}")
for symbol in subscribed_symbols:
symbol_changes.unsubscribe(symbol, ws)