async def main(): await objects.connect() logging.info("LT self_ws_source proc start...") mgr = WsManager() await mgr.run_forever()
async def receive_prize_from_udp_server(task_q: asyncio.Queue, broadcast_target: asyncio.coroutines): await mq_source_to_raffle.start_listen() while True: start_time = time.time() sources = [] try: while True: sources.append(mq_source_to_raffle.get_nowait()) except asyncio.queues.QueueEmpty: pass de_dup = set() for msg in sources: key_type, room_id, *_ = msg if key_type in ("R", "D", "P", "S", "A"): executor = Executor(start_time=start_time, br=broadcast_target) task_q.put_nowait(getattr(executor, key_type.lower())(*msg)) # logging.info(f"Assign task: {key_type} room_id: {room_id}") elif key_type in ("G", "T", "Z"): if room_id not in de_dup: de_dup.add(room_id) executor = Executor(start_time=start_time, br=broadcast_target) task_q.put_nowait(executor.lottery_or_guard(*msg)) logging.info(f"Assign task: {key_type} room_id: {room_id}") cost = time.time() - start_time if cost < 1: await asyncio.sleep(1 - cost)
async def get_live_rooms_from_api(): start = time.time() flag, total = await BiliApi.get_all_lived_room_count() if not flag: logging.error(f"Cannot get lived room count! msg: {total}") return target_count = min(total, MONITOR_COUNT) living_room_id_list = await batch_get_live_room_ids(count=target_count) api_cost = time.time() - start if abs(target_count - len(living_room_id_list)) > 1001: logging.error("从api获取的直播间数与目标差异过大,不予更新。") start = time.time() r = await MonitorLiveRooms.set(living_room_id_list) redis_cost = time.time() - start api_count = len(living_room_id_list) logging.info( f"MonitorLiveRooms set {api_count}, r: {r}, api cost: {api_cost:.3f}, redis_cost: {redis_cost:.3f}" ) established, time_wait = get_ws_established_and_time_wait() __monitor_info = { "api room cnt": len(living_room_id_list), "TCP ESTABLISHED": established, "TCP TIME_WAIT": time_wait, } await objects.connect() await MonitorWsClient.record(__monitor_info) await objects.close() return r
async def monitor_status(self): msg_speed_peak = 0 msg_count_of_last_second = 0 cyc_count = 0 cyc_duration = 59 while True: start_time = time.time() msg_speed_peak = max( self._message_count - msg_count_of_last_second, msg_speed_peak) msg_count_of_last_second = self._message_count if cyc_count % cyc_duration == 0: msg_speed_avg = self._message_count / cyc_duration log = f"Message speed avg: {msg_speed_avg:0.2f}, peak: {msg_speed_peak}. " active_clients = 0 monitor_rooms = set() for i, ws in enumerate(self._all_clients): monitor_rooms.add(ws.room_id) if ws.task and not ws.task.done(): active_clients += 1 else: logging.debug( f"\ti: {i} -> {ws.room_id}: task: {ws.task}, conn: {ws.ws_conn}, session: {ws.session}" ) broken_times = self._broken_clients.qsize() log += ( f"Clients {active_clients}/{len(self._all_clients)}, all rooms: {len(monitor_rooms)}, " f"broken_times: {broken_times}") broken_details = {} broken_count = 0 for _ in range(broken_times): room_id, reason = self._broken_clients.get_nowait().split( "$", 1) broken_details.setdefault(reason, []).append(room_id) for reason, rooms in broken_details.items(): broken_count += len(rooms) de_dup_rooms = set(rooms) log += f"\n\t{reason} {len(de_dup_rooms)} rooms broken {len(rooms)} times." logging.info(log) __monitor_info = { "msg speed": msg_speed_avg, "msg peak speed": msg_speed_peak, "broken clients": broken_count, "active clients": active_clients, "total clients": len(self._all_clients) } await MonitorWsClient.record(__monitor_info) self._message_count = 0 msg_speed_peak = 0 cost = time.time() - start_time await asyncio.sleep(max(0.0, 1 - cost)) cyc_count += 1
async def main(): logging.info(f"\n{'-' * 80}\nLT PROC_RAFFLE started!\n{'-' * 80}") await objects.connect() await mq_server.start_listen() processor = RaffleProcessor() await asyncio.gather(processor.receive(), processor.work())
async def task_print_info(self): count = 0 msg_count_of_last_second = 0 msg_speed_peak = 0 while True: msg_speed_peak = max(self.msg_count - msg_count_of_last_second, msg_speed_peak) msg_count_of_last_second = self.msg_count if count % 11 == 0: speed = self.msg_count / 11 if len(self._broken_live_rooms) > 50: append_msg = ( f"broken count: {len(self._broken_live_rooms)}, " f"{','.join([str(r) for r in self._broken_live_rooms[:10]])}" f"{' ...' if len(self._broken_live_rooms) > 10 else '.'}" ) else: append_msg = "" if msg_speed_peak < 20: logging.info( f"Message speed avg: {speed:0.2f}, peak: {msg_speed_peak}. {append_msg}" ) __monitor_info = { "msg speed": speed, "msg peak speed": msg_speed_peak, "broken clients": len(self._broken_live_rooms) } await MonitorWsClient.record(__monitor_info) self.msg_count = 0 self._broken_live_rooms = [] msg_count_of_last_second = 0 msg_speed_peak = 0 if count % 30 == 0: total = len(self._clients) valid_client_count = 0 for room_id, c in self._clients.items(): if c.status == "OPEN" and c.set_shutdown is False: valid_client_count += 1 if valid_client_count < total: logging.info( f"Active client count: {valid_client_count}, total: {total}." ) await MonitorWsClient.record({ "active clients": valid_client_count, "total clients": total }) count += 1 if count > 1000000000: count = 0 await asyncio.sleep(1)
async def _handle_tv(self, room_id, gift_list): await InLotteryLiveRooms.add(room_id=room_id) gift_type_to_name_map = {} for info in gift_list: raffle_id = info["raffleId"] key = f"T${room_id}${raffle_id}" if not await redis_cache.set_if_not_exists(key, "de-duplication"): continue gift_type = info["type"] gift_name = info.get("thank_text", "").split("赠送的", 1)[-1] created_time = datetime.datetime.fromtimestamp(self._start_time) expire_time = created_time + datetime.timedelta( seconds=info["time"]) bc = RaffleBroadCast( raffle_type="tv", ts=int(time.time()), real_room_id=room_id, raffle_id=raffle_id, gift_name=gift_name, created_time=created_time, expire_time=expire_time, gift_type=gift_type, time_wait=info["time_wait"], max_time=info["max_time"], ) await bc.save(redis_cache) sender_name = info["from_user"]["uname"] sender_face = info["from_user"]["face"] logging.info(f"\tLottery found: room_id: {room_id} $ {raffle_id} " f"({gift_name}) <- {sender_name}") create_param = { "raffle_id": raffle_id, "room_id": room_id, "gift_name": gift_name, "gift_type": gift_type, "sender_uid": None, "sender_name": sender_name, "sender_face": sender_face, "created_time": created_time, "expire_time": expire_time } await RedisRaffle.add(raffle_id=raffle_id, value=create_param, _pre=True) await Raffle.record_raffle_before_result(**create_param) gift_type_to_name_map[gift_type] = gift_name for gift_type, gift_name in gift_type_to_name_map.items(): await redis_cache.set(key=f"GIFT_TYPE_{gift_type}", value=gift_name)
async def get_one_page(page_no): for _try_times in range(3): if _try_times != 0: logging.info( f"get one page Failed: page no {page_no}, failed times: {_try_times}" ) flag, data = await BiliApi.get_lived_room_id_by_page( page=page_no, page_size=page_size, timeout=30) if not flag: await asyncio.sleep(1) continue result[page_no] = data return
async def receive(self): while True: de_dup = set() for index in range(mq_server.qzise()): msg = mq_server.get_nowait() if msg.prize_type in ("T", "Z"): if msg.room_id in de_dup: continue de_dup.add(msg.room_id) self._dmk_source_q.put_nowait(msg) logging.info( f"Assign task: {msg.prize_type} -> {msg.room_id}") elif msg.prize_type in ("G", "S", "R", "D", "P", "A", "RAFFLE_START"): self._dmk_source_q.put_nowait(msg) await asyncio.sleep(3)
async def monitor(index): my_area_id = index % 6 + 1 my_room_id = None while True: my_room_id = await get_living_room_id(index=index, area_id=my_area_id, old_room_id=my_room_id) await listen_ws(index=index, area_id=my_area_id, room_id=my_room_id) # print status message = "\n".join( sorted([ f"\t({index})area: {ws.area_id}, " f"room_id: {ws.monitor_room_id}, " f"closed ? {ws.closed}" for ws in ALL_WS_CLIENTS ])) logging.info(f"TV SOURCE STATUS (before updated.): \n{message}\n")
async def _handle_tv(self, room_id, gift_list): await InLotteryLiveRooms.add(room_id=room_id) gift_type_to_name_map = {} for info in gift_list: raffle_id = info["raffleId"] key = f"T${room_id}${raffle_id}" if not await redis_cache.set_if_not_exists(key, "de-duplication"): continue gift_type = info["type"] gift_name = info.get("thank_text", "").split("赠送的", 1)[-1] gift_type_to_name_map[gift_type] = gift_name await self.broadcast(json.dumps({ "raffle_type": "tv", "ts": int(self._start_time), "real_room_id": room_id, "raffle_id": raffle_id, "gift_name": gift_name, "gift_type": gift_type, "time_wait": info["time_wait"], "max_time": info["max_time"], }, ensure_ascii=False)) sender_name = info["from_user"]["uname"] sender_face = info["from_user"]["face"] created_time = datetime.datetime.fromtimestamp(self._start_time) logging.info(f"\tLottery found: room_id: {room_id} $ {raffle_id} ({gift_name}) <- {sender_name}") create_param = { "raffle_id": raffle_id, "room_id": room_id, "gift_name": gift_name, "gift_type": gift_type, "sender_uid": None, "sender_name": sender_name, "sender_face": sender_face, "created_time": created_time, "expire_time": created_time + datetime.timedelta(seconds=info["time"]) } await RedisRaffle.add(raffle_id=raffle_id, value=create_param, _pre=True) for gift_type, gift_name in gift_type_to_name_map.items(): await redis_cache.set(key=f"GIFT_TYPE_{gift_type}", value=gift_name)
async def _handle_guard(self, room_id, guard_list): for info in guard_list: raffle_id = info['id'] key = F"G${room_id}${raffle_id}" if not await redis_cache.set_if_not_exists(key, "de-duplication"): continue privilege = info["privilege_type"] gift_name = { 1: "舰长", 2: "提督", 3: "总督" }.get(privilege, f"guard_{privilege}") created_time = datetime.datetime.fromtimestamp(self._start_time) expire_time = created_time + datetime.timedelta( seconds=info["time"]) bc = RaffleBroadCast( raffle_type="guard", ts=int(time.time()), real_room_id=room_id, raffle_id=raffle_id, gift_name=gift_name, created_time=created_time, expire_time=expire_time, ) await bc.save(redis_cache) sender = info["sender"] create_param = { "gift_id": raffle_id, "room_id": room_id, "gift_name": gift_name, "sender_uid": sender["uid"], "sender_name": sender["uname"], "sender_face": sender["face"], "created_time": created_time, "expire_time": expire_time, } await RedisGuard.add(raffle_id=raffle_id, value=create_param) await Guard.create(**create_param) logging.info(f"\tGuard found: room_id: {room_id} $ {raffle_id} " f"({gift_name}) <- {sender['uname']}")
async def get_living_room_id(area_id, old_room_id): while True: flag, result = await BiliApi.get_living_rooms_by_area( area_id=area_id) if flag: for room_id in result: if room_id not in MONITOR_LIVE_ROOM_IDS: MONITOR_LIVE_ROOM_IDS.add(room_id) MONITOR_LIVE_ROOM_IDS.discard(old_room_id) logging.info( f"Get live rooms from Biliapi, {index}-{area_id}, old {old_room_id} -> {room_id}" ) return room_id else: logging.error( f"Cannot get live rooms from Biliapi. {index}-{area_id} -> {result}" ) await asyncio.sleep(30)
async def d(self, *args): """ danmaku to qq """ key_type, room_id, danmaku, *_ = args info = danmaku.get("info", {}) msg = str(info[1]) if msg in g.lottery_danmaku: return uid = info[2][0] user_name = info[2][1] is_admin = info[2][2] ul = info[4][0] d = info[3] dl = d[0] if d else "-" deco = d[1] if d else "undefined" message = ( f"{room_id} ({datetime.datetime.fromtimestamp(self._start_time)}) ->\n\n" f"{'[管] ' if is_admin else ''}[{deco} {dl}] [{uid}][{user_name}][{ul}]-> {msg}" ) logging.info(message) await async_zy.send_private_msg(user_id=g.QQ_NUMBER_DD, message=message)
async def listen_ws(index: int, area_id: int, room_id: int): is_preparing = False session = aiohttp.ClientSession() async with session.ws_connect(url=DANMAKU_WS_URL) as ws: await ws.send_bytes(WsApi.gen_join_room_pkg(room_id=room_id)) ws.monitor_room_id = room_id ws.area_id = area_id ALL_WS_CLIENTS.add(ws) async for msg in ws: if msg.type == aiohttp.WSMsgType.ERROR: break is_preparing = await proc_danmaku(area_id, room_id, msg.data) if is_preparing is True: break ALL_WS_CLIENTS.remove(ws) logging.info(f"Client closed. {index}-{area_id} -> {room_id}, " f"By danmaku preparing: {is_preparing}")
async def _handle_guard(self, room_id, guard_list): for info in guard_list: raffle_id = info['id'] key = F"G${room_id}${raffle_id}" if not await redis_cache.set_if_not_exists(key, "de-duplication"): continue privilege_type = info["privilege_type"] if privilege_type == 3: gift_name = "舰长" elif privilege_type == 2: gift_name = "提督" elif privilege_type == 1: gift_name = "总督" else: gift_name = f"guard_{privilege_type}" await self.broadcast(json.dumps({ "raffle_type": "guard", "ts": int(time.time()), "real_room_id": room_id, "raffle_id": raffle_id, "gift_name": gift_name, }, ensure_ascii=False)) created_time = datetime.datetime.fromtimestamp(self._start_time) expire_time = created_time + datetime.timedelta(seconds=info["time"]) sender = info["sender"] create_param = { "gift_id": raffle_id, "room_id": room_id, "gift_name": gift_name, "sender_uid": sender["uid"], "sender_name": sender["uname"], "sender_face": sender["face"], "created_time": created_time, "expire_time": expire_time, } await RedisGuard.add(raffle_id=raffle_id, value=create_param) logging.info(f"\tGuard found: room_id: {room_id} $ {raffle_id} ({gift_name}) <- {sender['uname']}")
async def get_living_room_id(index: int, area_id: int, old_room_id: int) -> int: while True: flag, result = await BiliApi.get_living_rooms_by_area(area_id=area_id) if not flag: logging.error(f"Cannot get live rooms from Biliapi. " f"{index}-{area_id} -> {result}") await asyncio.sleep(10) continue for room_id in result: if room_id not in MONITOR_LIVE_ROOM_IDS: MONITOR_LIVE_ROOM_IDS.add(room_id) MONITOR_LIVE_ROOM_IDS.discard(old_room_id) logging.info( f"Get live rooms from Biliapi, " f"{index}-{area_id}, old {old_room_id} -> {room_id}") return room_id await asyncio.sleep(30)
async def proc_danmaku(area_id, room_id, raw_msg) -> bool: for danmaku in WsApi.parse_msg(raw_msg): cmd = danmaku.get("cmd") if not cmd: continue if cmd in ("PREPARING", "ROOM_CHANGE"): return True elif cmd == "NOTICE_MSG": msg_type = danmaku.get("msg_type") if msg_type in (2, 8): real_room_id = danmaku['real_roomid'] await mq_client.put( DMKSource(prize_type="T", room_id=real_room_id)) logging.info(f"NOTICE_MSG received. room_id: {real_room_id}") elif cmd == "GUARD_MSG" and danmaku["buy_type"] == 1: prize_room_id = danmaku['roomid'] await mq_client.put( DMKSource(prize_type="Z", room_id=prize_room_id)) logging.info(f"GUARD_MSG received. room_id: {prize_room_id}") return False
async def main(): logging.info("-" * 80) logging.info("LT PROC_RAFFLE started!") logging.info("-" * 80) app = web.Application() app['ws'] = weakref.WeakSet() async def broadcaster(request): ws = web.WebSocketResponse() await ws.prepare(request) request.app['ws'].add(ws) try: async for msg in ws: pass finally: request.app['ws'].discard(ws) return ws app.add_routes([web.get('/raffle_wss', broadcaster), ]) runner = web.AppRunner(app) await runner.setup() site = web.TCPSite(runner, '127.0.0.1', 1024) await site.start() print("Site started.") async def broadcast_target(message): for ws in set(app['ws']): await ws.send_str(f"{message}\n") task_q = asyncio.Queue() receiver_task = asyncio.create_task(receive_prize_from_udp_server(task_q, broadcast_target)) async def worker(index): while True: c = await task_q.get() start_time = time.time() try: await c except Exception as e: logging.error(f"RAFFLE worker[{index}] error: {e}\n{traceback.format_exc()}") cost_time = time.time() - start_time if cost_time > 5: logging.warning(f"RAFFLE worker[{index}] exec long time: {cost_time:.3f}") await asyncio.gather(receiver_task, *[asyncio.create_task(worker(_)) for _ in range(8)])
async def process_one_danmaku(ts, room_id, msg): cmd = msg["cmd"] if cmd == "GUARD_BUY": await mq_source_to_raffle.put(("G", room_id)) logging.info(f"SOURCE: {cmd}, room_id: {room_id}") elif cmd == "PK_LOTTERY_START": await mq_source_to_raffle.put(("P", room_id, msg)) logging.info(f"SOURCE: {cmd}, room_id: {room_id}") elif cmd in ("RAFFLE_END", "TV_END", "ANCHOR_LOT_AWARD"): await mq_source_to_raffle.put(("R", room_id, msg)) display_msg = msg.get("data", {}).get("win", {}).get("msg", "") logging.info(f"SOURCE: {cmd}, room_id: {room_id}, msg: {display_msg}") elif cmd == "SEND_GIFT" and msg["data"]["giftName"] == "节奏风暴": await mq_source_to_raffle.put(("S", room_id)) logging.info(f"SOURCE: {cmd}-节奏风暴, room_id: {room_id}") elif cmd.startswith("DANMU_MSG"): if msg["info"][2][0] == 64782616: # uid = msg["info"][2][0] await mq_source_to_raffle.put(("D", room_id, msg)) logging.info( f"DANMU_MSG: put to mq, room_id: {room_id}, msg: {msg}") elif cmd == "ANCHOR_LOT_START": await mq_source_to_raffle.put(("A", room_id, msg)) data = msg["data"] logging.info( f"SOURCE: {cmd}, room_id: {room_id}, {data['require_text']} -> {data['award_name']}" )
async def update_connection(self): cyc_time = 60 while True: start_time = time.time() expected = await MonitorLiveRooms.get() in_lottery = await InLotteryLiveRooms.get_all() expected |= in_lottery valuable = await ValuableLiveRoom.get_all() valuable_hit_count = 0 for room_id in valuable: expected_len = len(expected) if expected_len >= MONITOR_COUNT: break expected.add(room_id) if len(expected) == expected_len: valuable_hit_count += 1 cache_hit_rate = valuable_hit_count / len(valuable) * 100 logging.debug(f"Monitor rooms get from redis: {len(expected)}") existed = {ws.room_id for ws in self._all_clients} need_add = expected - existed need_del = existed - expected if len(need_del) > 0 or len(need_add) > 0: log = f"WS MONITOR CLIENTS UPDATE: need add {len(need_add)}, del: {len(need_del)}" if 0 < len(need_add): log += f"\n\tadd: {list(need_add)[:10]}{'...' if len(need_add) > 10 else ''}" if 0 < len(need_del): log += f"\n\tdel: {list(need_del)[:10]}{'...' if len(need_del) > 10 else ''}" logging.info(log) need_del_clients = { ws for ws in self._all_clients if ws.room_id in need_del } for ws in need_del_clients: await ws.close() self._all_clients.remove(ws) for i, room_id in enumerate(need_add): if i > 0 and i % 300 == 0: await asyncio.sleep(1) async def on_message(msg, ws): self._message_count += 1 self._message_q.put_nowait((time.time(), ws.room_id, msg)) async def on_broken(reason, ws): self._broken_clients.put_nowait(f"{ws.room_id}${reason}") ws = WsClient( room_id=room_id, on_message=on_message, on_broken=on_broken, ) await ws.connect() self._all_clients.add(ws) # record __monitor_info = { "valuable room": len(valuable), "target clients": len(expected), "valuable hit rate": cache_hit_rate, } await MonitorWsClient.record(__monitor_info) cost = time.time() - start_time if cost < cyc_time: await asyncio.sleep(cyc_time - cost)
async def save(self, redis): result = await redis.zset_zadd(key=self.__key__, member_pairs=[(self, time.time())]) logging.info(f"RaffleBroadCast saved! {self} -> {result}")
async def main(): start = time.time() await get_live_rooms_from_api() logging.info( f"LT Flush monitor live room done, cost: {time.time() - start:.3f}.\n")
def parse(ts, room_id, msg): cmd = msg["cmd"] if cmd == "GUARD_LOTTERY_START": mq_source_to_raffle.put_nowait(("G", room_id, msg, ts)) logging.info(f"SOURCE: {cmd}, room_id: {room_id}") elif cmd == "SPECIAL_GIFT": mq_source_to_raffle.put_nowait(("S", room_id, msg, ts)) logging.info(f"SOURCE: {cmd}-节奏风暴, room_id: {room_id}") elif cmd == "PK_LOTTERY_START": mq_source_to_raffle.put_nowait(("P", room_id, msg, ts)) logging.info(f"SOURCE: {cmd}, room_id: {room_id}") elif cmd in ("RAFFLE_END", "TV_END", "ANCHOR_LOT_AWARD"): mq_source_to_raffle.put_nowait(("R", room_id, msg, ts)) display_msg = msg.get("data", {}).get("win", {}).get("msg", "") logging.info( f"SOURCE: {cmd}, room_id: {room_id}, msg: {display_msg}") elif cmd.startswith("DANMU_MSG"): if msg["info"][2][0] in ( 64782616, # 温柔桢 9859414, # G7 ): mq_source_to_raffle.put_nowait(("D", room_id, msg, ts)) logging.info( f"DANMU_MSG: put to mq, room_id: {room_id}, msg: {msg}") elif cmd == "ANCHOR_LOT_START": mq_source_to_raffle.put_nowait(("A", room_id, msg, ts)) data = msg["data"] logging.info( f"SOURCE: {cmd}, room_id: {room_id}, {data['require_text']} -> {data['award_name']}" ) elif cmd == "RAFFLE_START": data = msg["data"] mq_source_to_raffle.put_nowait(("RAFFLE_START", room_id, msg, ts)) logging.info( f"SOURCE: {cmd}, room_id: {room_id}, {data['thank_text']}")
async def monitor(index): async def proc_danmaku(area_id, room_id, raw_msg): for danmaku in WsApi.parse_msg(raw_msg): try: cmd = danmaku.get("cmd") if cmd == "PREPARING": return True elif cmd == "NOTICE_MSG": msg_type = danmaku.get("msg_type") if msg_type in (2, 8): real_room_id = danmaku['real_roomid'] await mq_source_to_raffle.put(("T", real_room_id)) elif cmd == "GUARD_MSG" and danmaku[ "buy_type"] == 1: # and area_id == 1: prize_room_id = danmaku[ 'roomid'] # TODO: need find real room id. await mq_source_to_raffle.put(("G", prize_room_id)) except KeyError: continue async def listen_ws(area_id, room_id): is_preparing = False session = aiohttp.ClientSession() async with session.ws_connect(url=DANMAKU_WS_URL) as ws: await ws.send_bytes(WsApi.gen_join_room_pkg(room_id=room_id)) ws.monitor_room_id = room_id ws.area_id = area_id ALL_WS_CLIENTS.add(ws) async for msg in ws: if msg.type == aiohttp.WSMsgType.ERROR: break is_preparing = await proc_danmaku(area_id, room_id, msg.data) if is_preparing is True: break ALL_WS_CLIENTS.remove(ws) logging.info( f"Client closed. {index}-{area_id} -> {room_id}, By danmaku preparing: {is_preparing}" ) async def get_living_room_id(area_id, old_room_id): while True: flag, result = await BiliApi.get_living_rooms_by_area( area_id=area_id) if flag: for room_id in result: if room_id not in MONITOR_LIVE_ROOM_IDS: MONITOR_LIVE_ROOM_IDS.add(room_id) MONITOR_LIVE_ROOM_IDS.discard(old_room_id) logging.info( f"Get live rooms from Biliapi, {index}-{area_id}, old {old_room_id} -> {room_id}" ) return room_id else: logging.error( f"Cannot get live rooms from Biliapi. {index}-{area_id} -> {result}" ) await asyncio.sleep(30) my_area_id = index % 6 + 1 my_room_id = None while True: my_room_id = await get_living_room_id(my_area_id, my_room_id) await listen_ws(area_id=my_area_id, room_id=my_room_id) # print status messages = [] for ws in ALL_WS_CLIENTS: messages.append( f"\t({index})area: {ws.area_id}, room_id: {ws.monitor_room_id}, closed ? {ws.closed}" ) message = "\n".join(sorted(messages)) logging.info( f"TV SOURCE CLIENTS STATUS (before updated.): \n{message}\n")
async def run_once(): start_time = time.time() logging.info(f"WS MONITOR CLIENTS UPDATING...start: {start_time}.") in_lottery = await InLotteryLiveRooms.get_all() expected = in_lottery | (await MonitorLiveRooms.get()) valuable = await ValuableLiveRoom.get_all() valuable_hit_count = 0 for room_id in valuable: expected_len = len(expected) if expected_len >= MONITOR_COUNT: break expected.add(room_id) if len(expected) == expected_len: valuable_hit_count += 1 cache_hit_rate = valuable_hit_count / len(valuable) * 100 existed = {ws.room_id for ws in self._all_clients} need_add = expected - existed need_del = existed - expected need_del_clients = { ws for ws in self._all_clients if ws.room_id in need_del } logging.info( f"WS MONITOR CLIENTS UPDATING: close non-active clients, count: {len(need_del_clients)}" ) for ws in need_del_clients: await ws.close() self._all_clients.remove(ws) logging.info(f"WS MONITOR CLIENTS CREATING NEW: {len(need_add)}") for i, room_id in enumerate(need_add): if i > 0 and i % 300 == 0: await asyncio.sleep(3) async def on_broken(reason, ws): self._broken_clients.put_nowait(f"{ws.room_id}${reason}") async def on_message(data, ws): self._message_count += 1 m = (int(time.time()), ws.room_id, data) self._message_q.put_nowait(m) ws = WsClient( room_id=room_id, on_message=on_message, on_broken=on_broken, ) await ws.connect() self._all_clients.add(ws) logging.info(f"WS MONITOR CLIENTS UPDATING: MonitorWsClient.") # record __monitor_info = { "valuable room": len(valuable), "target clients": len(expected), "valuable hit rate": cache_hit_rate, } await MonitorWsClient.record(__monitor_info) logging.info( f"WS MONITOR CLIENTS UPDATE! cost: {time.time() - start_time:.3f}." f"\n\tadd {len(need_add)}: {list(need_add)[:10]}" f"\n\tdel {len(need_del)}: {list(need_del)[:10]}" f"\n\texpected: {len(expected)}, in lottery {len(in_lottery)}, valuable: {len(valuable)}" )