class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ): time_interval_for_check_do_not_run_time = 60 BROKER_KIND = None @property @decorators.synchronized def publisher_of_same_queue(self): if not self._publisher_of_same_queue: self._publisher_of_same_queue = get_publisher(self._queue_name, consuming_function=self.consuming_function, broker_kind=self.BROKER_KIND) if self._msg_expire_senconds: self._publisher_of_same_queue.set_is_add_publish_time() return self._publisher_of_same_queue def bulid_a_new_publisher_of_same_queue(self): return get_publisher(self._queue_name, broker_kind=self.BROKER_KIND) @classmethod def join_shedual_task_thread(cls): """ :return: """ """ def ff(): RabbitmqConsumer('queue_test', consuming_function=f3, threads_num=20, msg_schedule_time_intercal=2, log_level=10, logger_prefix='yy平台消费', is_consuming_function_use_multi_params=True).start_consuming_message() RabbitmqConsumer('queue_test2', consuming_function=f4, threads_num=20, msg_schedule_time_intercal=4, log_level=10, logger_prefix='zz平台消费', is_consuming_function_use_multi_params=True).start_consuming_message() AbstractConsumer.join_shedual_task_thread() # 如果开多进程启动消费者,在linux上需要这样写下这一行。 if __name__ == '__main__': [Process(target=ff).start() for _ in range(4)] """ ConsumersManager.join_all_consumer_shedual_task_thread() # noinspection PyProtectedMember def __init__(self, queue_name, *, consuming_function: Callable = None, function_timeout=0, threads_num=50, concurrent_num=50, specify_threadpool=None, concurrent_mode=1, max_retry_times=3, log_level=10, is_print_detail_exception=True, msg_schedule_time_intercal=0.0, qps=0, msg_expire_senconds=0, logger_prefix='', create_logger_file=True, do_task_filtering=False, task_filtering_expire_seconds=0, is_consuming_function_use_multi_params=True, is_do_not_run_by_specify_time_effect=False, do_not_run_by_specify_time=('10:00:00', '22:00:00'), schedule_tasks_on_main_thread=False, function_result_status_persistance_conf=FunctionResultStatusPersistanceConfig( False, False, 7 * 24 * 3600), is_using_rpc_mode=False): """ :param queue_name: :param consuming_function: 处理消息的函数。 :param function_timeout : 超时秒数,函数运行超过这个时间,则自动杀死函数。为0是不限制。 :param threads_num:线程或协程并发数量 :param concurrent_num:并发数量,这个覆盖threads_num。以后会废弃threads_num参数,因为表达的意思不太准确,不一定是线程模式并发。 :param specify_threadpool:使用指定的线程池/携程池,可以多个消费者共使用一个线程池,不为None时候。threads_num失效 :param concurrent_mode:并发模式,暂时支持 线程 、gevent、eventlet三种模式。 1线程 2 gevent 3 evenlet :param max_retry_times: :param log_level: :param is_print_detail_exception: :param msg_schedule_time_intercal:消息调度的时间间隔,用于控频 :param qps:指定1秒内的函数执行次数,qps会覆盖msg_schedule_time_intercal,一会废弃msg_schedule_time_intercal这个参数。 :param logger_prefix: 日志前缀,可使不同的消费者生成不同的日志 :param create_logger_file : 是否创建文件日志 :param do_task_filtering :是否执行基于函数参数的任务过滤 :param task_filtering_expire_seconds:任务过滤的失效期,为0则永久性过滤任务。例如设置过滤过期时间是1800秒 , 30分钟前发布过1 + 2 的任务,现在仍然执行, 如果是30分钟以内发布过这个任务,则不执行1 + 2,现在把这个逻辑集成到框架,一般用于接口价格缓存。 :is_consuming_function_use_multi_params 函数的参数是否是传统的多参数,不为单个body字典表示多个参数。 :param is_do_not_run_by_specify_time_effect :是否使不运行的时间段生效 :param do_not_run_by_specify_time :不运行的时间段 :param schedule_tasks_on_main_thread :直接在主线程调度任务,意味着不能直接在当前主线程同时开启两个消费者。 :param function_result_status_persistance_conf :配置。是否保存函数的入参,运行结果和运行状态到mongodb。 这一步用于后续的参数追溯,任务统计和web展示,需要安装mongo。 :param is_using_rpc_mode 是否使用rpc模式,可以在发布端获取消费端的结果回调,但消耗一定性能,使用async_result.result时候会等待阻塞住当前线程。 """ ConsumersManager.consumers_queue__info_map[queue_name] = current_queue__info_dict = copy.copy(locals()) current_queue__info_dict['consuming_function'] = str(consuming_function) # consuming_function.__name__ current_queue__info_dict['function_result_status_persistance_conf'] = function_result_status_persistance_conf.to_dict() current_queue__info_dict.pop('self') current_queue__info_dict['broker_kind'] = self.__class__.BROKER_KIND current_queue__info_dict['class_name'] = self.__class__.__name__ concurrent_name = ConsumersManager.get_concurrent_name_by_concurrent_mode(concurrent_mode) current_queue__info_dict['concurrent_mode_name'] = concurrent_name # 方便点击跳转定位到当前解释器下所有实例化消费者的文件行,点击可跳转到该处。 # 获取被调用函数在被调用时所处代码行数 # 直接实例化相应的类和使用工厂模式来实例化相应的类,得到的消费者实际实例化的行是不一样的,希望定位到用户的代码处,而不是定位到工厂模式处。 line = sys._getframe(0).f_back.f_lineno # 获取被调用函数所在模块文件名 file_name = sys._getframe(1).f_code.co_filename if 'consumer_factory.py' in file_name: line = sys._getframe(1).f_back.f_lineno file_name = sys._getframe(2).f_code.co_filename current_queue__info_dict['where_to_instantiate'] = f'{file_name}:{line}' self._queue_name = queue_name self.queue_name = queue_name # 可以换成公有的,免得外部访问有警告。 self.consuming_function = consuming_function self._function_timeout = function_timeout self._threads_num = concurrent_num if threads_num == 50 else threads_num # concurrent参数优先,以后废弃threads_num参数。 self._specify_threadpool = specify_threadpool self._threadpool = None # 单独加一个检测消息数量和心跳的线程 self._concurrent_mode = concurrent_mode self._max_retry_times = max_retry_times self._is_print_detail_exception = is_print_detail_exception if qps != 0: msg_schedule_time_intercal = 1.0 / qps # 使用qps覆盖消息调度间隔,以qps为准,以后废弃msg_schedule_time_intercal这个参数。 self._msg_schedule_time_intercal = msg_schedule_time_intercal if msg_schedule_time_intercal > 0.001 else 0.001 self._msg_expire_senconds = msg_expire_senconds if self._concurrent_mode not in (1, 2, 3): raise ValueError('设置的并发模式不正确') self._concurrent_mode_dispatcher = ConcurrentModeDispatcher(self) self._logger_prefix = logger_prefix self._log_level = log_level if logger_prefix != '': logger_prefix += '--' logger_name = f'{logger_prefix}{self.__class__.__name__}--{concurrent_name}--{queue_name}' # nb_print(logger_name) self.logger = LogManager(logger_name).get_logger_and_add_handlers(log_level, log_filename=f'{logger_name}.log' if create_logger_file else None) # self.logger.info(f'{self.__class__} 在 {current_queue__info_dict["where_to_instantiate"]} 被实例化') sys.stdout.write(f'{time.strftime("%H:%M:%S")} "{current_queue__info_dict["where_to_instantiate"]}" \033[0;30;44m此行 实例化队列名 {current_queue__info_dict["queue_name"]} 的消费者, 类型为 {self.__class__}\033[0m\n') self._do_task_filtering = do_task_filtering self._redis_filter_key_name = f'filter_zset:{queue_name}' if task_filtering_expire_seconds else f'filter_set:{queue_name}' filter_class = RedisFilter if task_filtering_expire_seconds == 0 else RedisImpermanencyFilter self._redis_filter = filter_class(self._redis_filter_key_name, task_filtering_expire_seconds) self._is_consuming_function_use_multi_params = is_consuming_function_use_multi_params self._execute_task_times_every_minute = 0 # 每分钟执行了多少次任务。 self._lock_for_count_execute_task_times_every_minute = Lock() self._current_time_for_execute_task_times_every_minute = time.time() self._msg_num_in_broker = 0 self._last_timestamp_when_has_task_in_queue = 0 self._last_timestamp_print_msg_num = 0 self._is_do_not_run_by_specify_time_effect = is_do_not_run_by_specify_time_effect self._do_not_run_by_specify_time = do_not_run_by_specify_time # 可以设置在指定的时间段不运行。 self._schedule_tasks_on_main_thread = schedule_tasks_on_main_thread self._result_persistence_helper = ResultPersistenceHelper(function_result_status_persistance_conf, queue_name) self._is_using_rpc_mode = is_using_rpc_mode self.stop_flag = False self._publisher_of_same_queue = None self.custom_init() @property @decorators.synchronized def threadpool(self): return self._concurrent_mode_dispatcher.build_pool() def custom_init(self): pass def keep_circulating(self, time_sleep=0.001, exit_if_function_run_sucsess=False, is_display_detail_exception=True, block=True): """间隔一段时间,一直循环运行某个方法的装饰器 :param time_sleep :循环的间隔时间 :param is_display_detail_exception :param exit_if_function_run_sucsess :如果成功了就退出循环 :param block:是否阻塞在当前主线程运行。 """ def _keep_circulating(func): @wraps(func) def __keep_circulating(*args, **kwargs): # noinspection PyBroadException def ___keep_circulating(): while 1: try: result = func(*args, **kwargs) if exit_if_function_run_sucsess: return result except Exception as e: msg = func.__name__ + ' 运行出错\n ' + traceback.format_exc(limit=10) if is_display_detail_exception else str(e) self.logger.error(msg) finally: time.sleep(time_sleep) if block: return ___keep_circulating() else: threading.Thread(target=___keep_circulating, ).start() return __keep_circulating return _keep_circulating def start_consuming_message(self): self.logger.warning(f'开始消费 {self._queue_name} 中的消息') self.keep_circulating(20, block=False)(self.check_heartbeat_and_message_count)() self._redis_filter.delete_expire_filter_task_cycle() if self._schedule_tasks_on_main_thread: self.keep_circulating(1)(self._shedual_task)() else: self._concurrent_mode_dispatcher.schedulal_task_with_no_block() @abc.abstractmethod def _shedual_task(self): """ 每个子类必须实现这个的方法,完成如何从中间件取出消息,并将函数和运行参数添加到工作池。 :return: """ raise NotImplementedError def _run(self, kw: dict, ): do_task_filtering_priority = self.__get_priority_conf(kw, 'do_task_filtering') function_only_params = delete_keys_and_return_new_dict(kw['body'], ) if do_task_filtering_priority and self._redis_filter.check_value_exists(function_only_params): # 对函数的参数进行检查,过滤已经执行过并且成功的任务。 self.logger.info(f'redis的 [{self._redis_filter_key_name}] 键 中 过滤任务 {kw["body"]}') self._confirm_consume(kw) return with self._lock_for_count_execute_task_times_every_minute: self._execute_task_times_every_minute += 1 if time.time() - self._current_time_for_execute_task_times_every_minute > 60: self.logger.info( f'一分钟内执行了 {self._execute_task_times_every_minute} 次函数 [ {self.consuming_function.__name__} ] ,预计' f'还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / self._execute_task_times_every_minute * 60)} 时间' f'才能执行完成 {self._msg_num_in_broker}个剩余的任务 ') self._current_time_for_execute_task_times_every_minute = time.time() self._execute_task_times_every_minute = 0 self._run_consuming_function_with_confirm_and_retry(kw, current_retry_times=0, function_result_status=FunctionResultStatus( self.queue_name, self.consuming_function.__name__, kw['body']), do_task_filtering_priority=do_task_filtering_priority) def __get_priority_conf(self, kw: dict, broker_task_config_key: str): broker_task_config = kw['body'].get('extra', {}).get(broker_task_config_key, None) if broker_task_config is None: return getattr(self, f'_{broker_task_config_key}') else: return broker_task_config def _run_consuming_function_with_confirm_and_retry(self, kw: dict, current_retry_times, function_result_status: FunctionResultStatus, do_task_filtering_priority): function_only_params = delete_keys_and_return_new_dict(kw['body']) if current_retry_times < self.__get_priority_conf(kw, 'max_retry_times'): function_result_status.run_times += 1 # noinspection PyBroadException t_start = time.time() try: function_run = self.consuming_function if self._function_timeout == 0 else self._concurrent_mode_dispatcher.timeout_deco(self.__get_priority_conf(kw, 'function_timeout'))(self.consuming_function) if self._is_consuming_function_use_multi_params: # 消费函数使用传统的多参数形式 function_result_status.result = function_run(**function_only_params) else: function_result_status.result = function_run(function_only_params) # 消费函数使用单个参数,参数自身是一个字典,由键值对表示各个参数。 function_result_status.success = True self._confirm_consume(kw) if do_task_filtering_priority: self._redis_filter.add_a_value(function_only_params) # 函数执行成功后,添加函数的参数排序后的键值对字符串到set中。 self.logger.debug(f' 函数 {self.consuming_function.__name__} ' f'第{current_retry_times + 1}次 运行, 正确了,函数运行时间是 {round(time.time() - t_start, 4)} 秒,入参是 【 {function_only_params} 】。 {ConsumersManager.get_concurrent_info()}') except Exception as e: if isinstance(e, (PyMongoError, ExceptionForRequeue)): # mongo经常维护备份时候插入不了或挂了,或者自己主动抛出一个ExceptionForRequeue类型的错误会重新入队,不受指定重试次数逇约束。 self.logger.critical(f'函数 [{self.consuming_function.__name__}] 中发生错误 {type(e)} {e}') return self._requeue(kw) self.logger.error(f'函数 {self.consuming_function.__name__} 第{current_retry_times + 1}次发生错误,' f'函数运行时间是 {round(time.time() - t_start, 4)} 秒,\n 入参是 【 {function_only_params} 】 \n 原因是 {type(e)} {e} ', exc_info=self.__get_priority_conf(kw, 'is_print_detail_exception')) function_result_status.exception = f'{e.__class__.__name__} {str(e)}' self._run_consuming_function_with_confirm_and_retry(kw, current_retry_times + 1, function_result_status, do_task_filtering_priority) else: self.logger.critical(f'函数 {self.consuming_function.__name__} 达到最大重试次数 {self.__get_priority_conf(kw, "max_retry_times")} 后,仍然失败, 入参是 【 {function_only_params} 】') self._confirm_consume(kw) # 错得超过指定的次数了,就确认消费了。 self._result_persistence_helper.save_function_result_to_mongo(function_result_status) if self.__get_priority_conf(kw, 'is_using_rpc_mode'): # print(function_result_status.get_status_dict(without_datetime_obj=True)) with RedisMixin().redis_db_frame.pipeline() as p: # RedisMixin().redis_db_frame.lpush(kw['body']['extra']['task_id'], json.dumps(function_result_status.get_status_dict(without_datetime_obj=True))) # RedisMixin().redis_db_frame.expire(kw['body']['extra']['task_id'], 600) p.lpush(kw['body']['extra']['task_id'], json.dumps(function_result_status.get_status_dict(without_datetime_obj=True))) p.expire(kw['body']['extra']['task_id'], 600) p.execute() @abc.abstractmethod def _confirm_consume(self, kw): """确认消费""" raise NotImplementedError def check_heartbeat_and_message_count(self): self._msg_num_in_broker = self.publisher_of_same_queue.get_message_count() if time.time() - self._last_timestamp_print_msg_num > 60: self.logger.info(f'[{self._queue_name}] 队列中还有 [{self._msg_num_in_broker}] 个任务') self._last_timestamp_print_msg_num = time.time() if self._msg_num_in_broker != 0: self._last_timestamp_when_has_task_in_queue = time.time() return self._msg_num_in_broker @abc.abstractmethod def _requeue(self, kw): """重新入队""" raise NotImplementedError def _submit_task(self, kw): if self._judge_is_daylight(): self._requeue(kw) time.sleep(self.time_interval_for_check_do_not_run_time) return publish_time = _get_publish_time(kw['body']) msg_expire_senconds_priority = self.__get_priority_conf(kw, 'msg_expire_senconds') if msg_expire_senconds_priority != 0 and time.time() - msg_expire_senconds_priority > publish_time: self.logger.warning(f'消息发布时戳是 {publish_time} {kw["body"].get("publish_time_format", "")},距离现在 {round(time.time() - publish_time, 4)} 秒 ,' f'超过了指定的 {msg_expire_senconds_priority} 秒,丢弃任务') self._confirm_consume(kw) return 0 self.threadpool.submit(self._run, kw) time.sleep(self._msg_schedule_time_intercal) @decorators.FunctionResultCacher.cached_function_result_for_a_time(120) def _judge_is_daylight(self): if self._is_do_not_run_by_specify_time_effect and self._do_not_run_by_specify_time[0] < time_util.DatetimeConverter().time_str < self._do_not_run_by_specify_time[1]: self.logger.warning(f'现在时间是 {time_util.DatetimeConverter()} ,现在时间是在 {self._do_not_run_by_specify_time} 之间,不运行') return True def __str__(self): return f'队列为 {self.queue_name} 函数为 {self.consuming_function} 的消费者'
class AbstractConsumer( LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ): time_interval_for_check_do_not_run_time = 60 BROKER_KIND = None @property @decorators.synchronized def publisher_of_same_queue(self): if not self._publisher_of_same_queue: self._publisher_of_same_queue = get_publisher( self._queue_name, broker_kind=self.BROKER_KIND) if self._msg_expire_senconds: self._publisher_of_same_queue.set_is_add_publish_time() return self._publisher_of_same_queue @classmethod def join_shedual_task_thread(cls): """ :return: """ """ def ff(): RabbitmqConsumer('queue_test', consuming_function=f3, threads_num=20, msg_schedule_time_intercal=2, log_level=10, logger_prefix='yy平台消费', is_consuming_function_use_multi_params=True).start_consuming_message() RabbitmqConsumer('queue_test2', consuming_function=f4, threads_num=20, msg_schedule_time_intercal=4, log_level=10, logger_prefix='zz平台消费', is_consuming_function_use_multi_params=True).start_consuming_message() AbstractConsumer.join_shedual_task_thread() # 如果开多进程启动消费者,在linux上需要这样写下这一行。 if __name__ == '__main__': [Process(target=ff).start() for _ in range(4)] """ ConsumersManager.join_all_consumer_shedual_task_thread() def __init__( self, queue_name, *, consuming_function: Callable = None, function_timeout=0, threads_num=50, specify_threadpool=None, concurrent_mode=1, max_retry_times=3, log_level=10, is_print_detail_exception=True, msg_schedule_time_intercal=0.0, msg_expire_senconds=0, logger_prefix='', create_logger_file=True, do_task_filtering=False, is_consuming_function_use_multi_params=True, is_do_not_run_by_specify_time_effect=False, do_not_run_by_specify_time=('10:00:00', '22:00:00'), schedule_tasks_on_main_thread=False, function_result_status_persistance_conf=FunctionResultStatusPersistanceConfig( False, False, 7 * 24 * 3600)): """ :param queue_name: :param consuming_function: 处理消息的函数。 :param function_timeout : 超时秒数,函数运行超过这个时间,则自动杀死函数。为0是不限制。 :param threads_num: :param specify_threadpool:使用指定的线程池,可以多个消费者共使用一个线程池,不为None时候。threads_num失效 :param concurrent_mode:并发模式,暂时支持 线程 、gevent、eventlet三种模式。 1线程 2 gevent 3 evenlet :param max_retry_times: :param log_level: :param is_print_detail_exception: :param msg_schedule_time_intercal:消息调度的时间间隔,用于控频 :param logger_prefix: 日志前缀,可使不同的消费者生成不同的日志 :param create_logger_file : 是否创建文件日志 :param do_task_filtering :是否执行基于函数参数的任务过滤 :is_consuming_function_use_multi_params 函数的参数是否是传统的多参数,不为单个body字典表示多个参数。 :param is_do_not_run_by_specify_time_effect :是否使不运行的时间段生效 :param do_not_run_by_specify_time :不运行的时间段 :param schedule_tasks_on_main_thread :直接在主线程调度任务,意味着不能直接在当前主线程同时开启两个消费者。 :function_result_status_persistance_conf :配置。是否保存函数的入参,运行结果和运行状态到mongodb。这一步用于后续的参数追溯, 任务统计和web展示,需要安装mongo。 """ ConsumersManager.consumers_queue__info_map[ queue_name] = current_queue__info_dict = copy.copy(locals()) current_queue__info_dict['consuming_function'] = str( consuming_function) # consuming_function.__name__ current_queue__info_dict[ 'function_result_status_persistance_conf'] = function_result_status_persistance_conf.to_dict( ) current_queue__info_dict.pop('self') current_queue__info_dict['broker_kind'] = self.__class__.BROKER_KIND current_queue__info_dict['class_name'] = self.__class__.__name__ concurrent_name = ConsumersManager.get_concurrent_name_by_concurrent_mode( concurrent_mode) current_queue__info_dict['concurrent_mode_name'] = concurrent_name self._queue_name = queue_name self.queue_name = queue_name # 可以换成公有的,免得外部访问有警告。 self.consuming_function = consuming_function self._function_timeout = function_timeout self._threads_num = threads_num self._specify_threadpool = specify_threadpool self._threadpool = None # 单独加一个检测消息数量和心跳的线程 self._concurrent_mode = concurrent_mode self._max_retry_times = max_retry_times self._is_print_detail_exception = is_print_detail_exception self._msg_schedule_time_intercal = msg_schedule_time_intercal if msg_schedule_time_intercal > 0.001 else 0.001 self._msg_expire_senconds = msg_expire_senconds if self._concurrent_mode not in (1, 2, 3): raise ValueError('设置的并发模式不正确') self._concurrent_mode_dispatcher = ConcurrentModeDispatcher(self) self._logger_prefix = logger_prefix self._log_level = log_level if logger_prefix != '': logger_prefix += '--' logger_name = f'{logger_prefix}{self.__class__.__name__}--{concurrent_name}--{queue_name}' # nb_print(logger_name) self.logger = LogManager(logger_name).get_logger_and_add_handlers( log_level, log_filename=f'{logger_name}.log' if create_logger_file else None) self.logger.info(f'{self.__class__} 被实例化') self._do_task_filtering = do_task_filtering self._redis_filter_key_name = f'filter:{queue_name}' self._redis_filter = RedisFilter(self._redis_filter_key_name) self._is_consuming_function_use_multi_params = is_consuming_function_use_multi_params self._lock_for_pika = Lock() self._execute_task_times_every_minute = 0 # 每分钟执行了多少次任务。 self._lock_for_count_execute_task_times_every_minute = Lock() self._current_time_for_execute_task_times_every_minute = time.time() self._msg_num_in_broker = 0 self._last_timestamp_when_has_task_in_queue = 0 self._last_timestamp_print_msg_num = 0 self._is_do_not_run_by_specify_time_effect = is_do_not_run_by_specify_time_effect self._do_not_run_by_specify_time = do_not_run_by_specify_time # 可以设置在指定的时间段不运行。 self._schedule_tasks_on_main_thread = schedule_tasks_on_main_thread self._result_persistence_helper = ResultPersistenceHelper( function_result_status_persistance_conf, queue_name) self.stop_flag = False self._publisher_of_same_queue = None self.custom_init() @property @decorators.synchronized def threadpool(self): return self._concurrent_mode_dispatcher.build_pool() def custom_init(self): pass def keep_circulating(self, time_sleep=0.001, exit_if_function_run_sucsess=False, is_display_detail_exception=True): """间隔一段时间,一直循环运行某个方法的装饰器 :param time_sleep :循环的间隔时间 :param is_display_detail_exception :param exit_if_function_run_sucsess :如果成功了就退出循环 """ def _keep_circulating(func): # noinspection PyBroadException @wraps(func) def __keep_circulating(*args, **kwargs): while 1: if self.stop_flag: break try: result = func(*args, **kwargs) if exit_if_function_run_sucsess: return result except Exception as e: msg = func.__name__ + ' 运行出错\n ' + traceback.format_exc( limit=10) if is_display_detail_exception else str( e) self.logger.error(msg) finally: time.sleep(time_sleep) return __keep_circulating return _keep_circulating def start_consuming_message(self): self.logger.warning(f'开始消费 {self._queue_name} 中的消息') # self.threadpool.submit(decorators.keep_circulating(20)(self.check_heartbeat_and_message_count)) self.threadpool.submit( self.keep_circulating(20)(self.check_heartbeat_and_message_count)) if self._schedule_tasks_on_main_thread: # decorators.keep_circulating(1)(self._shedual_task)() self.keep_circulating(1)(self._shedual_task)() else: # t = Thread(target=decorators.keep_circulating(1)(self._shedual_task)) self._concurrent_mode_dispatcher.schedulal_task_with_no_block() @abc.abstractmethod def _shedual_task(self): raise NotImplementedError def _run( self, kw: dict, ): if self._do_task_filtering and self._redis_filter.check_value_exists( kw['body']): # 对函数的参数进行检查,过滤已经执行过并且成功的任务。 self.logger.info( f'redis的 [{self._redis_filter_key_name}] 键 中 过滤任务 {kw["body"]}' ) self._confirm_consume(kw) return with self._lock_for_count_execute_task_times_every_minute: self._execute_task_times_every_minute += 1 if time.time( ) - self._current_time_for_execute_task_times_every_minute > 60: self.logger.info( f'一分钟内执行了 {self._execute_task_times_every_minute} 次函数 [ {self.consuming_function.__name__} ] ,预计' f'还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / self._execute_task_times_every_minute * 60)} 时间' f'才能执行完成 {self._msg_num_in_broker}个剩余的任务 ') self._current_time_for_execute_task_times_every_minute = time.time( ) self._execute_task_times_every_minute = 0 self._run_consuming_function_with_confirm_and_retry( kw, current_retry_times=0, function_result_status=FunctionResultStatus( self.queue_name, self.consuming_function.__name__, kw['body'])) def _run_consuming_function_with_confirm_and_retry( self, kw: dict, current_retry_times, function_result_status: FunctionResultStatus): if current_retry_times < self._max_retry_times: function_result_status.run_times += 1 # noinspection PyBroadException t_start = time.time() try: function_run = self.consuming_function if self._function_timeout == 0 else self._concurrent_mode_dispatcher.timeout_deco( self._function_timeout)(self.consuming_function) if self._is_consuming_function_use_multi_params: # 消费函数使用传统的多参数形式 function_result_status.result = function_run( **delete_keys_and_return_new_dict( kw['body'], ['publish_time', 'publish_time_format'])) else: function_result_status.result = function_run( delete_keys_and_return_new_dict( kw['body'], ['publish_time', 'publish_time_format' ])) # 消费函数使用单个参数,参数自身是一个字典,由键值对表示各个参数。 function_result_status.success = True self._confirm_consume(kw) if self._do_task_filtering: self._redis_filter.add_a_value( kw['body']) # 函数执行成功后,添加函数的参数排序后的键值对字符串到set中。 self.logger.debug( f' 函数 {self.consuming_function.__name__} ' f'第{current_retry_times + 1}次 运行, 正确了,函数运行时间是 {round(time.time() - t_start, 4)} 秒,入参是 【 {kw["body"]} 】。 {ConsumersManager.get_concurrent_info()}' ) except Exception as e: if isinstance( e, (PyMongoError, ExceptionForRequeue) ): # mongo经常维护备份时候插入不了或挂了,或者自己主动抛出一个ExceptionForRequeue类型的错误会重新入队,不受指定重试次数逇约束。 self.logger.critical( f'函数 [{self.consuming_function.__name__}] 中发生错误 {type(e)} {e}' ) return self._requeue(kw) self.logger.error( f'函数 {self.consuming_function.__name__} 第{current_retry_times + 1}次发生错误,' f'函数运行时间是 {round(time.time() - t_start, 4)} 秒,\n 入参是 【 {kw["body"]} 】 \n 原因是 {type(e)} {e} ', exc_info=self._is_print_detail_exception) function_result_status.exception = f'{e.__class__.__name__} {str(e)}' self._run_consuming_function_with_confirm_and_retry( kw, current_retry_times + 1, function_result_status) else: self.logger.critical( f'函数 {self.consuming_function.__name__} 达到最大重试次数 {self._max_retry_times} 后,仍然失败, 入参是 【 {kw["body"]} 】' ) self._confirm_consume(kw) # 错得超过指定的次数了,就确认消费了。 self._result_persistence_helper.save_function_result_to_mongo( function_result_status) @abc.abstractmethod def _confirm_consume(self, kw): """确认消费""" raise NotImplementedError # noinspection PyUnusedLocal def check_heartbeat_and_message_count(self): self._msg_num_in_broker = self.publisher_of_same_queue.get_message_count( ) if time.time() - self._last_timestamp_print_msg_num > 60: self.logger.info( f'[{self._queue_name}] 队列中还有 [{self._msg_num_in_broker}] 个任务') self._last_timestamp_print_msg_num = time.time() if self._msg_num_in_broker != 0: self._last_timestamp_when_has_task_in_queue = time.time() return self._msg_num_in_broker @abc.abstractmethod def _requeue(self, kw): """重新入队""" raise NotImplementedError def _submit_task(self, kw): if self._judge_is_daylight(): self._requeue(kw) time.sleep(self.time_interval_for_check_do_not_run_time) return if self._msg_expire_senconds != 0 and time.time( ) - self._msg_expire_senconds > kw['body']['publish_time']: self.logger.warning( f'消息发布时戳是 {kw["body"]["publish_time"]} {kw["body"].get("publish_time_format", "")},距离现在 {round(time.time() - kw["body"]["publish_time"], 4)} 秒 ,' f'超过了指定的 {self._msg_expire_senconds} 秒,丢弃任务') self._confirm_consume(kw) return 0 self.threadpool.submit(self._run, kw) time.sleep(self._msg_schedule_time_intercal) @decorators.FunctionResultCacher.cached_function_result_for_a_time(120) def _judge_is_daylight(self): if self._is_do_not_run_by_specify_time_effect and self._do_not_run_by_specify_time[ 0] < time_util.DatetimeConverter( ).time_str < self._do_not_run_by_specify_time[1]: self.logger.warning( f'现在时间是 {time_util.DatetimeConverter()} ,现在时间是在 {self._do_not_run_by_specify_time} 之间,不运行' ) return True def __str__(self): return f'队列为 {self.queue_name} 函数为 {self.consuming_function} 的消费者'