def processing_task(self, key, queue_length):
"""
按任务优先级处理数据
:param key: 任务队列key值
:param queue_length: 当前队列长度
:return:
"""
"""读取最大量"""
max_redis_read_length = BusiBaseConfService().search_key_return_value(
(busi_config.MAX_REDIS_READ_LENGTH))
self.logger.info("单次最大处理量为：%s" % max_redis_read_length)
"""读取redis列表"""
result = BusinessUtil.get_redis_lpop_by_key_and_length(
key, queue_length, int(max_redis_read_length))
"""判空"""
if not result:
return
"""最大线程数"""
thread_max_workers = BusiBaseConfService().search_key_return_value(
(busi_config.THREAD_MAX_WORKERS))
"""按发送邮箱名称分组"""
task_dict = {}
with ThreadPoolExecutor(
max_workers=int(thread_max_workers)) as executor:
future_list = []
"""循环"""
for item in result:
if item:
future = executor.submit(self.task_thread_submit, item,
key, task_dict)
future_list.append(future)
"""等待子线程结束"""
wait(future_list, timeout=10)
"""处理队列数据"""
self.deal_with_result_data(task_dict)
def redis_instance(self):
"""
启动redis消费
:return:
"""
try:
"""获取订阅信息"""
kafka_topic = "get_%s_queue" % self.task_choice_pattern.replace(
".", "_")
"""读取当前redis内数据长度"""
queue_length = BusinessUtil.get_redis_llen_by_key(kafka_topic)
"""读取最大量"""
consume_max_redis_read_length = BusiBaseConfService(
).search_key_return_value(
(busi_config.CONSUME_MAX_REDIS_READ_LENGTH))
self.logger.info("单次最大处理量为：%s" % consume_max_redis_read_length)
"""读取redis列表"""
result = BusinessUtil.get_redis_lpop_by_key_and_length(
kafka_topic, queue_length, int(consume_max_redis_read_length))
"""判空"""
if not result:
self.logger.info("【消费】未读取到有效数据，流程结束")
return
"""处理消息"""
self.processing_task(result)
"""处理结束"""
self.logger.info("【消费】处理结束")
except Exception as e:
self.logger.error("【消费】启动消费异常，信息为：%s" % traceback.format_exc())