def join_all_consumer_shedual_task_thread(cls): nb_print((cls.schedulal_thread_to_be_join, len(cls.schedulal_thread_to_be_join), '模式:', cls.global_concurrent_mode)) if cls.schedual_task_always_use_thread: for t in cls.schedulal_thread_to_be_join: nb_print(t) t.join() else: if cls.global_concurrent_mode == 1: for t in cls.schedulal_thread_to_be_join: nb_print(t) t.join() elif cls.global_concurrent_mode == 2: # cls.logger.info() nb_print(cls.schedulal_thread_to_be_join) gevent.joinall( cls.schedulal_thread_to_be_join, raise_error=True, ) elif cls.global_concurrent_mode == 3: for g in cls.schedulal_thread_to_be_join: # eventlet.greenthread.GreenThread. nb_print(g) g.wait()
def wait_for_possible_has_finish_all_tasks(queue_name: str, minutes: int, send_stop_to_broker=0, broker_kind: int = 0, ): """ 由于是异步消费,和存在队列一边被消费,一边在推送,或者还有结尾少量任务还在确认消费者实际还没彻底运行完成。 但有时候需要判断 所有任务,务是否完成,提供一个不精确的判断,要搞清楚原因和场景后再慎用。 :param queue_name: 队列名字 :param minutes: 连续多少分钟没任务就判断为消费已完成 :param send_stop_to_broker :发送停止标志到中间件,这回导致消费退出循环调度。 :param broker_kind: 中间件种类 :return: """ if minutes <= 1: raise ValueError('疑似完成任务,判断时间最少需要设置为2分钟内,最好是是10分钟') pb = get_publisher(queue_name, broker_kind=broker_kind) no_task_time = 0 while 1: # noinspection PyBroadException try: message_count = pb.get_message_count() except Exception as e: nb_print(e) message_count = -1 if message_count == 0: no_task_time += 30 else: no_task_time = 0 time.sleep(30) if no_task_time > minutes * 60: break if send_stop_to_broker: pb.publish({'stop': 1}) pb.close()
def show_all_consumer_info(cls): nb_print( f'当前解释器内,所有消费者的信息是:\n {json.dumps(cls.consumers_queue__info_map, indent=4, ensure_ascii=False)}' ) for _, consumer_info in cls.consumers_queue__info_map.items(): sys.stdout.write( f'{time.strftime("%H:%M:%S")} "{consumer_info["where_to_instantiate"]}" \033[0;30;44m{consumer_info["queue_name"]} 的消费者\033[0m\n' )
def __gevent_timeout_deceo(*args, **kwargs): timeout = gevent.Timeout(timeout_t, ) timeout.start() try: f(*args, **kwargs) except gevent.Timeout as t: logger_gevent_timeout_deco.error(f'函数 {f} 运行超过了 {timeout_t} 秒') if t is not timeout: nb_print(t) # raise # not my timeout finally: timeout.close()
def __evenlet_timeout_deco(*args, **kwargs): timeout = Timeout(timeout_t, ) # timeout.start() # 与gevent不一样,直接start了。 try: f(*args, **kwargs) except Timeout as t: logger_evenlet_timeout_deco.error( f'函数 {f} 运行超过了 {timeout_t} 秒') if t is not timeout: nb_print(t) # raise # not my timeout finally: timeout.cancel()
def format_logs_dir_to_multi(logs_dir): parent_dir_list = list() pa = Path(f'/{logs_dir}') while True: nb_print(pa.as_posix()) parent_dir_list.append({ 'url': url_for('index', logs_dir=pa.as_posix()[1:]), 'dir_name': pa.name[:] }) pa = pa.parent if pa == Path('/'): parent_dir_list.append({ 'url': url_for('index', logs_dir=''), 'dir_name': '根目录' }) break nb_print(parent_dir_list) return parent_dir_list
def show_frame_config(): nb_print('显示当前的项目中间件配置参数') for var_name in dir(frame_config): if var_name.isupper(): var_value = getattr(frame_config, var_name) if var_name == 'MONGO_CONNECT_URL': if re.match('mongodb://.*?:.*?@.*?/.*', var_value): mongo_pass = re.search('mongodb://.*?:(.*?)@', var_value).group(1) mongo_pass_encryption = f'{"*" * (len(mongo_pass) - 2)}{mongo_pass[-1]}' if len(mongo_pass) > 3 else mongo_pass var_value_encryption = re.sub(r':(\w+)@', f':{mongo_pass_encryption}@', var_value) nb_print(f'{var_name}: {var_value_encryption}') continue if 'PASS' in var_name and len(var_value) > 3: # 对密码打* nb_print(f'{var_name}: {var_value[0]}{"*" * (len(var_value) - 2)}{var_value[-1]}') else: nb_print(f'{var_name}: {var_value}')
def show_frame_config(): nb_print('显示当前的项目中间件配置参数') for var_name in dir(frame_config): if var_name.isupper(): var_value = getattr(frame_config, var_name) if 'PASS' in var_name and len(var_value) > 3: # 对密码打* nb_print(f'{var_name}: {var_value[0]}{"*" * (len(var_value) - 2)}{var_value[-1]}') else: nb_print(f'{var_name}: {var_value}')
return threading.active_count() if __name__ == '__main__': from function_scheduling_distributed_framework.utils import decorators from function_scheduling_distributed_framework.concurrent_pool.bounded_threadpoolexcutor import BoundedThreadPoolExecutor # @decorators.keep_circulating(1) def f1(a): time.sleep(0.2) nb_print(f'{a} 。。。。。。。') # raise Exception('抛个错误测试') # show_current_threads_num() pool = CustomThreadPoolExecutor(200).set_log_level(10).set_min_workers() # pool = BoundedThreadPoolExecutor(200) # 测试对比原来写的BoundedThreadPoolExecutor show_current_threads_num(sleep_time=5) for i in range(300): time.sleep( 0.3 ) # 这里的间隔时间模拟,当任务来临不密集,只需要少量线程就能搞定f1了,因为f1的消耗时间短,不需要开那么多线程,CustomThreadPoolExecutor比BoundedThreadPoolExecutor 优势之一。 pool.submit(f1, str(i)) nb_print(6666) # pool.shutdown(wait=True) pool.submit(f1, 'yyyy') # 下面测试阻塞主线程退出的情况。注释掉可以测主线程退出的情况。 while True: time.sleep(10)
def __init__(self, x, y, z): in_param = copy.deepcopy(locals()) nb_print(f'执行初始化啦, {in_param}')
import sys import threading import time import traceback import unittest from functools import wraps # noinspection PyUnresolvedReferences import pysnooper from tomorrow3 import threads as tomorrow_threads from function_scheduling_distributed_framework.utils import LogManager, nb_print # noinspection PyUnresolvedReferences from function_scheduling_distributed_framework.utils.custom_pysnooper import _snoop_can_click, snoop_deco, patch_snooper_max_variable_length os_name = os.name nb_print(f' 操作系统类型是 {os_name}') handle_exception_log = LogManager('function_error').get_logger_and_add_handlers() run_times_log = LogManager('run_many_times').get_logger_and_add_handlers(20) class CustomException(Exception): def __init__(self, err=''): err0 = 'fatal exception\n' Exception.__init__(self, err0 + err) def run_many_times(times=1): """把函数运行times次的装饰器 :param times:运行次数 没有捕获错误,出错误就中断运行,可以配合handle_exception装饰器不管是否错误都运行n次。 """
def f2(x): time.sleep(3) nb_print(x)
def show_all_consumer_info(cls): nb_print( f'当前解释器内,所有消费者的信息是:\n {json.dumps(cls.consumers_queue__info_map, indent=4, ensure_ascii=False)}' )
class CustomEventletPoolExecutor(greenpool.GreenPool): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) check_evenlet_monkey_patch() atexit.register(self.shutdown) def submit(self, *args, **kwargs): # 保持为一直的公有用法。 # nb_print(args) self.spawn_n(*args, **kwargs) # self.spawn_n(*args, **kwargs) def shutdown(self): self.waitall() if __name__ == '__main__': # greenpool.GreenPool.waitall() monkey_patch(all=True) def f2(x): time.sleep(2) nb_print(x) pool = CustomEventletPoolExecutor(4) for i in range(15): nb_print(f'放入{i}') pool.submit(evenlet_timeout_deco(8)(f2), i)
def _show_current_threads_num(): while True: # logger_show_current_threads_num.info(f'{process_name} 进程 的 并发数量是 --> {threading.active_count()}') nb_print( f'{process_name} 进程 的 线程数量是 --> {threading.active_count()}') time.sleep(sleep_time)
def f1(a): time.sleep(0.2) nb_print(f'{a} 。。。。。。。')
except Exception as exc: self.logger.exception( f'函数 {fn.__name__} 中发生错误,错误原因是 {type(exc)} {exc} ') finally: pass self._q.task_done() def submit(self, fn: Callable, *args, **kwargs): # self.logger.debug(self._q.qsize()) self._q.put((fn, args, kwargs)) def __atexit(self): self.logger.critical('想即将退出程序。') self._q.join() if __name__ == '__main__': monkey.patch_all() def f2(x): time.sleep(3) nb_print(x) pool = GeventPoolExecutor(4) for i in range(20): nb_print(f'放入{i}') pool.submit(gevent_timeout_deco(0.8)(f2), i) nb_print(66666666)