async def update_minute_price(symbol, time, price, volume): # datetime(2020, 5, 18, 16, 55, 33) -> datetime(2020, 5, 18, 16, 56, 00) rounded_time = time.replace(second=0, microsecond=0) if rounded_time < time: # pretty always rounded_time += timedelta(seconds=60) while True: try: return await MINUTE_COLLECTION.update_one( { "symbol": symbol, "time": rounded_time, }, { "$set": {"updated": time, "close": price, "volume": volume}, "$min": {"low": price}, "$max": {"high": price}, "$setOnInsert": {"open": price}, "$inc": {"updates": 1}, }, upsert=True ) except PyMongoError as e: logger.error(f"Update stat exc {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"}) await asyncio.sleep(MONGODB_ERROR_INTERVAL)
async def get_minute_stats(symbol, limit=3000): collection = get_mongodb_collection(collection_name=MONGODB_MINUTE_COLLECTION) result = [] try: cursor = collection.find( {"symbol": symbol}, ) async for obj in cursor.limit(limit): obj["id"] = str(obj.pop("_id")) del obj["symbol"] result.append(obj) except PyMongoError as e: logger.error(f"Mongodb error {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"}) raise web.HTTPInternalServerError() else: return {"symbol": symbol, "data": result}
async def get_symbols(offset=0, limit=3000): collection = get_mongodb_collection(collection_name=MONGODB_COLLECTION) result = [] query = {} try: cursor = collection.find( query, {"name": 1} ) async for obj in cursor.skip(offset).limit(limit): result.append(dict(id=obj["_id"], name=obj["name"])) except PyMongoError as e: logger.error(f"Mongodb error {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"}) raise web.HTTPInternalServerError() else: return result
async def main(symbols): if not symbols: symbols = [s["id"] for s in await get_symbols()] conn = aiohttp.TCPConnector(ttl_dns_cache=300) async with aiohttp.ClientSession(connector=conn, headers=WS_SESSION_HEADER) as session: ws = await session.ws_connect(WS_ENDPOINT) msg_data = {"subscribe": symbols} await ws.send_str(json.dumps(msg_data)) logger.info("Sent sub", extra={"MESSAGE": msg_data}) while RUN: msg = await ws.receive() if msg.type == aiohttp.WSMsgType.TEXT: await process_message(msg.data) else: logger.error(f"Unexpected msg.type {msg}") break
async def incoming_listener(ws, queue, functions): subscribed_symbols = set() try: while not ws.closed: incoming = await ws.receive() logger.info("Got incoming msg", extra={"MESSAGE": incoming}) if incoming.type is WSMsgType.TEXT: try: data = json.loads(incoming.data) except ValueError as e: logger.error(e) else: new_functions = data.get("functions") if new_functions: # the same obj, but new contents functions[:] = new_functions subscribe = data.get("subscribe") if subscribe: subscribe = set(subscribe) delete_symbols = subscribed_symbols - subscribe for symbol in delete_symbols: symbol_changes.unsubscribe(symbol, ws) add_symbols = subscribe - subscribed_symbols for symbol in add_symbols: symbol_changes.subscribe(symbol, ws, queue) logger.info( f"Subscribed to {add_symbols} and unsubscribed from {delete_symbols}" ) for symbol in add_symbols: start_data = await get_minute_stats(symbol) queue.put_nowait(start_data) except ConnectionResetError as e: logger.warning(f"ConnectionResetError while getting data {e}") finally: logger.info(f"Unsubscribe socket from {subscribed_symbols}") for symbol in subscribed_symbols: symbol_changes.unsubscribe(symbol, ws)
async def watch_changes_minute_stats(): collection = get_mongodb_collection(collection_name=MONGODB_MINUTE_COLLECTION) resume_after = None while True: logger.info(f"Start watching mongodb changes after {resume_after}") changes = collection.watch(full_document="updateLookup", resume_after=resume_after) while True: try: change = await changes.next() except PyMongoError as e: logger.error(f"Got feed error {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"}) await asyncio.sleep(MONGODB_ERROR_INTERVAL) except StopAsyncIteration as e: logger.exception(e) resume_after = None break except Exception as e: logger.exception(e) await asyncio.sleep(MONGODB_ERROR_INTERVAL) else: resume_after = change["_id"] if "fullDocument" in change: # present in insert/update changes yield change["fullDocument"]