def join_all_consumer_shedual_task_thread(cls):
nb_print((cls.schedulal_thread_to_be_join,
len(cls.schedulal_thread_to_be_join), '模式：',
cls.global_concurrent_mode))
if cls.schedual_task_always_use_thread:
for t in cls.schedulal_thread_to_be_join:
nb_print(t)
t.join()
else:
if cls.global_concurrent_mode == 1:
for t in cls.schedulal_thread_to_be_join:
nb_print(t)
t.join()
elif cls.global_concurrent_mode == 2:
# cls.logger.info()
nb_print(cls.schedulal_thread_to_be_join)
gevent.joinall(
cls.schedulal_thread_to_be_join,
raise_error=True,
)
elif cls.global_concurrent_mode == 3:
for g in cls.schedulal_thread_to_be_join:
# eventlet.greenthread.GreenThread.
nb_print(g)
g.wait()
def wait_for_possible_has_finish_all_tasks(queue_name: str, minutes: int, send_stop_to_broker=0, broker_kind: int = 0, ):
"""
由于是异步消费，和存在队列一边被消费，一边在推送，或者还有结尾少量任务还在确认消费者实际还没彻底运行完成。 但有时候需要判断 所有任务，务是否完成，提供一个不精确的判断，要搞清楚原因和场景后再慎用。
:param queue_name: 队列名字
:param minutes: 连续多少分钟没任务就判断为消费已完成
:param send_stop_to_broker :发送停止标志到中间件，这回导致消费退出循环调度。
:param broker_kind: 中间件种类
:return:
"""
if minutes <= 1:
raise ValueError('疑似完成任务，判断时间最少需要设置为2分钟内,最好是是10分钟')
pb = get_publisher(queue_name, broker_kind=broker_kind)
no_task_time = 0
while 1:
# noinspection PyBroadException
try:
message_count = pb.get_message_count()
except Exception as e:
nb_print(e)
message_count = -1
if message_count == 0:
no_task_time += 30
else:
no_task_time = 0
time.sleep(30)
if no_task_time > minutes * 60:
break
if send_stop_to_broker:
pb.publish({'stop': 1})
pb.close()
def show_all_consumer_info(cls):
nb_print(
f'当前解释器内，所有消费者的信息是：\n {json.dumps(cls.consumers_queue__info_map, indent=4, ensure_ascii=False)}'
)
for _, consumer_info in cls.consumers_queue__info_map.items():
sys.stdout.write(
f'{time.strftime("%H:%M:%S")} "{consumer_info["where_to_instantiate"]}" \033[0;30;44m{consumer_info["queue_name"]} 的消费者\033[0m\n'
)
def __gevent_timeout_deceo(*args, **kwargs):
timeout = gevent.Timeout(timeout_t, )
timeout.start()
try:
f(*args, **kwargs)
except gevent.Timeout as t:
logger_gevent_timeout_deco.error(f'函数 {f} 运行超过了 {timeout_t} 秒')
if t is not timeout:
nb_print(t)
# raise # not my timeout
finally:
timeout.close()
def __evenlet_timeout_deco(*args, **kwargs):
timeout = Timeout(timeout_t, )
# timeout.start() # 与gevent不一样,直接start了。
try:
f(*args, **kwargs)
except Timeout as t:
logger_evenlet_timeout_deco.error(
f'函数 {f} 运行超过了 {timeout_t} 秒')
if t is not timeout:
nb_print(t)
# raise # not my timeout
finally:
timeout.cancel()
def format_logs_dir_to_multi(logs_dir):
parent_dir_list = list()
pa = Path(f'/{logs_dir}')
while True:
nb_print(pa.as_posix())
parent_dir_list.append({
'url':
url_for('index', logs_dir=pa.as_posix()[1:]),
'dir_name':
pa.name[:]
})
pa = pa.parent
if pa == Path('/'):
parent_dir_list.append({
'url': url_for('index', logs_dir=''),
'dir_name': '根目录'
})
break
nb_print(parent_dir_list)
return parent_dir_list
def show_frame_config():
nb_print('显示当前的项目中间件配置参数')
for var_name in dir(frame_config):
if var_name.isupper():
var_value = getattr(frame_config, var_name)
if var_name == 'MONGO_CONNECT_URL':
if re.match('mongodb://.*?:.*?@.*?/.*', var_value):
mongo_pass = re.search('mongodb://.*?:(.*?)@', var_value).group(1)
mongo_pass_encryption = f'{"*" * (len(mongo_pass) - 2)}{mongo_pass[-1]}' if len(mongo_pass) > 3 else mongo_pass
var_value_encryption = re.sub(r':(\w+)@', f':{mongo_pass_encryption}@', var_value)
nb_print(f'{var_name}: {var_value_encryption}')
continue
if 'PASS' in var_name and len(var_value) > 3: # 对密码打*
nb_print(f'{var_name}: {var_value[0]}{"*" * (len(var_value) - 2)}{var_value[-1]}')
else:
nb_print(f'{var_name}: {var_value}')
def show_frame_config():
nb_print('显示当前的项目中间件配置参数')
for var_name in dir(frame_config):
if var_name.isupper():
var_value = getattr(frame_config, var_name)
if 'PASS' in var_name and len(var_value) > 3: # 对密码打*
nb_print(f'{var_name}: {var_value[0]}{"*" * (len(var_value) - 2)}{var_value[-1]}')
else:
nb_print(f'{var_name}: {var_value}')
return threading.active_count()
if __name__ == '__main__':
from function_scheduling_distributed_framework.utils import decorators
from function_scheduling_distributed_framework.concurrent_pool.bounded_threadpoolexcutor import BoundedThreadPoolExecutor
# @decorators.keep_circulating(1)
def f1(a):
time.sleep(0.2)
nb_print(f'{a} 。。。。。。。')
# raise Exception('抛个错误测试')
# show_current_threads_num()
pool = CustomThreadPoolExecutor(200).set_log_level(10).set_min_workers()
# pool = BoundedThreadPoolExecutor(200) # 测试对比原来写的BoundedThreadPoolExecutor
show_current_threads_num(sleep_time=5)
for i in range(300):
time.sleep(
0.3
) # 这里的间隔时间模拟，当任务来临不密集，只需要少量线程就能搞定f1了，因为f1的消耗时间短，不需要开那么多线程，CustomThreadPoolExecutor比BoundedThreadPoolExecutor 优势之一。
pool.submit(f1, str(i))
nb_print(6666)
# pool.shutdown(wait=True)
pool.submit(f1, 'yyyy')
# 下面测试阻塞主线程退出的情况。注释掉可以测主线程退出的情况。
while True:
time.sleep(10)
def __init__(self, x, y, z):
in_param = copy.deepcopy(locals())
nb_print(f'执行初始化啦, {in_param}')
import sys
import threading
import time
import traceback
import unittest
from functools import wraps
# noinspection PyUnresolvedReferences
import pysnooper
from tomorrow3 import threads as tomorrow_threads
from function_scheduling_distributed_framework.utils import LogManager, nb_print
# noinspection PyUnresolvedReferences
from function_scheduling_distributed_framework.utils.custom_pysnooper import _snoop_can_click, snoop_deco, patch_snooper_max_variable_length
os_name = os.name
nb_print(f' 操作系统类型是 {os_name}')
handle_exception_log = LogManager('function_error').get_logger_and_add_handlers()
run_times_log = LogManager('run_many_times').get_logger_and_add_handlers(20)
class CustomException(Exception):
def __init__(self, err=''):
err0 = 'fatal exception\n'
Exception.__init__(self, err0 + err)
def run_many_times(times=1):
"""把函数运行times次的装饰器
:param times:运行次数
没有捕获错误，出错误就中断运行，可以配合handle_exception装饰器不管是否错误都运行n次。
"""
def f2(x):
time.sleep(3)
nb_print(x)
def show_all_consumer_info(cls):
nb_print(
f'当前解释器内，所有消费者的信息是：\n {json.dumps(cls.consumers_queue__info_map, indent=4, ensure_ascii=False)}'
)
class CustomEventletPoolExecutor(greenpool.GreenPool):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
check_evenlet_monkey_patch()
atexit.register(self.shutdown)
def submit(self, *args, **kwargs): # 保持为一直的公有用法。
# nb_print(args)
self.spawn_n(*args, **kwargs)
# self.spawn_n(*args, **kwargs)
def shutdown(self):
self.waitall()
if __name__ == '__main__':
# greenpool.GreenPool.waitall()
monkey_patch(all=True)
def f2(x):
time.sleep(2)
nb_print(x)
pool = CustomEventletPoolExecutor(4)
for i in range(15):
nb_print(f'放入{i}')
pool.submit(evenlet_timeout_deco(8)(f2), i)
def _show_current_threads_num():
while True:
# logger_show_current_threads_num.info(f'{process_name} 进程 的 并发数量是 --> {threading.active_count()}')
nb_print(
f'{process_name} 进程 的 线程数量是 --> {threading.active_count()}')
time.sleep(sleep_time)
def f1(a):
time.sleep(0.2)
nb_print(f'{a} 。。。。。。。')
except Exception as exc:
self.logger.exception(
f'函数 {fn.__name__} 中发生错误，错误原因是 {type(exc)} {exc} ')
finally:
pass
self._q.task_done()
def submit(self, fn: Callable, *args, **kwargs):
# self.logger.debug(self._q.qsize())
self._q.put((fn, args, kwargs))
def __atexit(self):
self.logger.critical('想即将退出程序。')
self._q.join()
if __name__ == '__main__':
monkey.patch_all()
def f2(x):
time.sleep(3)
nb_print(x)
pool = GeventPoolExecutor(4)
for i in range(20):
nb_print(f'放入{i}')
pool.submit(gevent_timeout_deco(0.8)(f2), i)
nb_print(66666666)
