def run_deploy() -> None: """ 定时任务,建议使用if而非for构造任务线程池 @return: """ try: # 初始化任务对象 dockers = [] # 载入定时任务权限配置 tasks = ENABLE_DEPLOY['tasks'] for docker_name, permission in tasks.items(): if permission: dockers.append({ "name": docker_name, "api": eval(f"_cd.startup_{docker_name}") }) # 无论有无权限都要装载采集器 if not tasks['collector']: dockers.append({ "name": 'collector', "api": _cd.startup_collector }) # 启动定时任务 GeventSchedule(dockers=dockers) except KeyError: logger.critical('config中枢层配置被篡改,ENABLE_DEPLOY 配置中无”tasks“键值对') exit() except NameError: logger.critical('eval()或exec()语法异常,检测变量名是否不一致。')
def sync_launch_interval() -> dict: # 读取配置文件 launch_interval = LAUNCH_INTERVAL # 检查配置并返回修正过后的任务配置 for task_name, task_interval in launch_interval.items(): # 未填写或填写异常数字 if (not task_interval) or (task_interval <= 1): logger.critical( f"<launch_interval>--{task_name}设置出现致命错误,即将熔断线程。间隔为空或小于1") raise Exception # 填写浮点数 if not isinstance(task_interval, int): logger.warning( f"<launch_interval>--{task_name}任务间隔应为整型int,参数已拟合") # 尝试类型转换若不中则赋一个默认值 60s try: launch_interval.update({task_name: int(task_interval)}) except TypeError: launch_interval.update({task_name: 60}) # 填写过小的任务间隔数,既设定的发动频次过高,主动拦截并修正为最低容错 60s/run if task_interval < 60: logger.warning( f"<launch_interval>--{task_name}任务频次过高,应不少于60/次,参数已拟合") launch_interval.update({task_name: 60}) else: return launch_interval
def offload_task(self): for key_ in self.keys: try: for sub, _ in self.rc.hgetall(key_).items(): self.work_q.put_nowait([sub, key_]) except redis_error.ResponseError: logger.critical("Link pool is broken down.")
def _del_subs(self, key_: str, subs: str, err_) -> None: try: self.rc.hdel(key_, subs) terminal_echo(f"detach -> {subs} {err_}", 3) except redis_error.ConnectionError: logger.critical( "<SubscribeCleaner> The local network communication is abnormal." )
def run_deploy() -> None: """ 定时任务,建议使用if而非for构造任务线程池 @return: """ # 载入定时任务权限配置 tasks = ENABLE_DEPLOY['tasks'] task2function = { 'ddt_decouple': _cd.startup_ddt_decouple, 'ddt_overdue': _cd.startup_ddt_overdue, } try: # 初始化调度器 docker_of_based_scheduler = TasksScheduler() docker_of_collector_scheduler = CollectorScheduler() # 清洗配置 使调度间隔更加合理 interval = _cd.sync_launch_interval() # 添加任务 for docker_name, permission in tasks.items(): logger.info( f"[Job] {docker_name} -- interval: {interval[docker_name]}s -- run: {permission}" ) # 若开启采集器则使用CollectorScheduler映射任务 # 使用久策略将此分流判断注释既可 if docker_name == "collector": docker_of_collector_scheduler.mapping_config({ 'interval': interval[docker_name], 'permission': permission, }) continue if permission: docker_of_based_scheduler.add_job({ "name": docker_name, "api": task2function[docker_name], 'interval': interval[docker_name], 'permission': True }) # 启动定时任务 要求执行采集任务时必须至少携带另一种其他部署任务 docker_of_collector_scheduler.deploy_jobs() docker_of_based_scheduler.deploy_jobs() except ConnectionError: logger.warning( "<RedisIO> Network communication failure, please check the network connection." ) except KeyError: logger.critical(f'config中枢层配置被篡改,ENABLE_DEPLOY 配置中无对应键值对{tasks}') sys.exit() except NameError: logger.critical('eval()或exec()语法异常,检测变量名是否不一致。')
def _sync_launch_interval() -> dict: # 热读取配置文件 launch_interval = LAUNCH_INTERVAL for check in launch_interval.items(): if not check[-1] or check[-1] <= 1: logger.critical( f"<launch_interval>--{check[0]}设置出现致命错误,即将熔断线程。间隔为空或小于1") raise Exception if not isinstance(check[-1], int): logger.warning( f"<launch_interval>--{check[0]}任务间隔应为整型int,参数已拟合") launch_interval.update({check[0]: round(check[-1])}) if check[-1] < 60: logger.warning( f"<launch_interval>--{check[0]}任务频次过高,应不少于60/次,参数已拟合") launch_interval.update({check[0]: 60}) else: return launch_interval
def get(self, key_name, pop_=0) -> str or bool: """ 分发订阅链接 每次get请求都会强制关闭连接 @param pop_: @param key_name: 任务类型,用于定位 redis hash name @return: """ try: while True: # 一次性抽取key_name(hashName)下的所有subscribe target_raw: dict = self.db.hgetall(key_name) try: # 弹出并捕获 <离当前时间> <最远一次入库>的订阅连接 既订阅链接并未按end_life排序 self.subscribe, end_life = list( target_raw.items()).pop(pop_) # end_life: requests_time(采集时间点) + vip_crontab(机场会员时长(账号试用时长)) # 该模块最初设计时仅针对<vip_crontab = 1day>的机场采集 # 后续又将试用几天甚至几个月的机场加入了任务队列 # 但该分发逻辑并未更新 以后版本将继续沿用这套分发逻辑 # 若之后版本的项目支持end_life动态更新时(既加入某种技术能够实时反馈机场主对会员规则的修改) # 此分发逻辑将会加入排序功能 # 若链接过期 -> loop next -> finally :db-del stale subscribe if self.is_stale(end_life, beyond=3): continue # 若链接可用 -> break off -> 分发 -> finally :db-del subscribe else: return self.subscribe # 出现该错误视为redis队列被击穿 无任何可用的链接分发,中断循环 except IndexError: logger.critical("{}.get() IndexError".format( self.__class__.__name__)) return False # 关联解除 finally: from src.BusinessCentralLayer.middleware.subscribe_io import detach detach(self.subscribe, beat_sync=True) finally: # 关闭连接 self.kill()
def control_driver(self, sub_info: List[str]): """ @param sub_info: [subs,key_secret_class] @return: """ try: # 解耦指定簇 if self.kill_ and self.kill_ in sub_info[0]: self._del_subs(sub_info[-1], sub_info[0], "target") else: # 解析订阅 node_info: dict = subs2node(sub_info[0], False) # 打印debug信息 if self.debug: print( f"check -- {node_info['subs']} -- {node_info['node'].__len__()}" ) # 订阅解耦 if node_info['node'].__len__() <= 4: self._del_subs(sub_info[-1], sub_info[0], "decouple") except UnicodeDecodeError or TypeError as e: logger.debug( f"Retry put the subscribe({sub_info}) to work queue -- {e}") # 单个链接重试3次,标记超时链接 if self.temp_cache.get(sub_info[0]): self.temp_cache[sub_info[0]] += 1 else: self.temp_cache[sub_info[0]] = 1 if self.temp_cache[sub_info[0]] <= 3: self.work_q.put_nowait(sub_info) else: self._del_subs(sub_info[-1], sub_info[0], e) except SystemExit: logger.critical("请关闭系统代理后再执行订阅清洗操作") except Exception as e: logger.warning(f"{sub_info} -- {e}") self._del_subs(sub_info[-1], sub_info[0])