def queue_json_publish(queue_name, event, processor, call_consume_in_tests=False): # type: (str, Union[Dict[str, Any], str], Callable[[Any], None], bool) -> None # most events are dicts, but zerver.middleware.write_log_line uses a str with queue_lock: if settings.USING_RABBITMQ: get_queue_client().json_publish(queue_name, event) elif call_consume_in_tests: # Must be imported here: A top section import leads to obscure not-defined-ish errors. from zerver.worker.queue_processors import get_worker get_worker(queue_name).consume_wrapper(event) # type: ignore # https://github.com/python/mypy/issues/3360 else: processor(event)
def queue_json_publish(queue_name: str, event: Dict[str, Any], processor: Callable[[Any], None] = None) -> None: # most events are dicts, but zerver.middleware.write_log_line uses a str with queue_lock: if settings.USING_RABBITMQ: get_queue_client().json_publish(queue_name, event) elif processor: processor(event) else: # Must be imported here: A top section import leads to obscure not-defined-ish errors. from zerver.worker.queue_processors import get_worker get_worker(queue_name).consume_wrapper(event)
def queue_json_publish(queue_name: str, event: Union[Dict[str, Any], str], processor: Callable[[Any], None]=None) -> None: # most events are dicts, but zerver.middleware.write_log_line uses a str with queue_lock: if settings.USING_RABBITMQ: get_queue_client().json_publish(queue_name, event) elif processor: processor(event) else: # Must be imported here: A top section import leads to obscure not-defined-ish errors. from zerver.worker.queue_processors import get_worker get_worker(queue_name).consume_wrapper(event)
def queue_json_publish( queue_name: str, event: Dict[str, Any], processor: Optional[Callable[[Any], None]] = None, ) -> None: if settings.USING_RABBITMQ: get_queue_client().json_publish(queue_name, event) elif processor: processor(event) else: # Must be imported here: A top section import leads to circular imports from zerver.worker.queue_processors import get_worker get_worker(queue_name).consume_single_event(event)
def queue_json_publish( queue_name: str, event: Dict[str, Any], processor: Optional[Callable[[Any], None]] = None, ) -> None: with queue_lock: if settings.USING_RABBITMQ: get_queue_client().json_publish(queue_name, event) elif processor: processor(event) else: # Must be imported here: A top section import leads to obscure not-defined-ish errors. from zerver.worker.queue_processors import get_worker get_worker(queue_name).consume_wrapper(event)
def __init__(self, queue_name: str, logger: logging.Logger) -> None: threading.Thread.__init__(self) self.logger = logger self.queue_name = queue_name with log_and_exit_if_exception(logger, queue_name, threaded=True): self.worker = get_worker(queue_name)
def handle(self, *args: Any, **options: Any) -> None: logging.basicConfig() logger = logging.getLogger("process_queue") def exit_with_three(signal: int, frame: Optional[FrameType]) -> None: """ This process is watched by Django's autoreload, so exiting with status code 3 will cause this process to restart. """ logger.warning("SIGUSR1 received. Restarting this queue processor.") sys.exit(3) if not settings.USING_RABBITMQ: # Make the warning silent when running the tests if settings.TEST_SUITE: logger.info("Not using RabbitMQ queue workers in the test suite.") else: logger.error("Cannot run a queue processor when USING_RABBITMQ is False!") raise CommandError def run_threaded_workers(queues: List[str], logger: logging.Logger) -> None: cnt = 0 for queue_name in queues: if not settings.DEVELOPMENT: logger.info("launching queue worker thread %s", queue_name) cnt += 1 td = ThreadedWorker(queue_name, logger) td.start() assert len(queues) == cnt logger.info("%d queue worker threads were launched", cnt) if options["all"]: signal.signal(signal.SIGUSR1, exit_with_three) autoreload.run_with_reloader(run_threaded_workers, get_active_worker_queues(), logger) elif options["multi_threaded"]: signal.signal(signal.SIGUSR1, exit_with_three) queues = options["multi_threaded"] autoreload.run_with_reloader(run_threaded_workers, queues, logger) else: queue_name = options["queue_name"] worker_num = options["worker_num"] def signal_handler(signal: int, frame: Optional[FrameType]) -> None: logger.info("Worker %d disconnecting from queue %s", worker_num, queue_name) worker.stop() sys.exit(0) logger.info("Worker %d connecting to queue %s", worker_num, queue_name) with log_and_exit_if_exception(logger, queue_name, threaded=False): worker = get_worker(queue_name) with configure_scope() as scope: scope.set_tag("queue_worker", queue_name) scope.set_tag("worker_num", worker_num) worker.setup() signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGUSR1, signal_handler) worker.ENABLE_TIMEOUTS = True worker.start()
def handle(self, *args, **options): logging.basicConfig() logger = logging.getLogger('process_queue') queue_name = options['queue_name'] worker_num = options['worker_num'] if not settings.USING_RABBITMQ: logger.error( "Cannot run a queue processor when USING_RABBITMQ is False!") sys.exit(1) logger.info("Worker %d connecting to queue %s" % (worker_num, queue_name)) worker = get_worker(queue_name) def signal_handler(signal, frame): logger.info("Worker %d disconnecting from queue %s" % (worker_num, queue_name)) worker.stop() sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) worker.start()
def handle(self, *args, **options): logging.basicConfig() logger = logging.getLogger('process_queue') if not settings.USING_RABBITMQ: logger.error("Cannot run a queue processor when USING_RABBITMQ is False!") sys.exit(1) if options['all']: for queue_name in get_active_worker_queues(): logger.info('launching queue worker thread ' + queue_name) td = Threaded_worker(queue_name) td.start() else: queue_name = options['queue_name'] worker_num = options['worker_num'] logger.info("Worker %d connecting to queue %s" % (worker_num, queue_name)) worker = get_worker(queue_name) worker.setup() def signal_handler(signal, frame): logger.info("Worker %d disconnecting from queue %s" % (worker_num, queue_name)) worker.stop() sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) worker.start()
def handle(self, *args, **options): # type: (*Any, **Any) -> None logging.basicConfig() logger = logging.getLogger('process_queue') def exit_with_three(signal, frame): # type: (int, FrameType) -> None """ This process is watched by Django's autoreload, so exiting with status code 3 will cause this process to restart. """ logger.warn("SIGUSR1 received. Restarting this queue processor.") sys.exit(3) if not settings.USING_RABBITMQ: # Make the warning silent when running the tests if settings.TEST_SUITE: logger.info("Not using RabbitMQ queue workers in the test suite.") else: logger.error("Cannot run a queue processor when USING_RABBITMQ is False!") sys.exit(1) def run_threaded_workers(queues, logger): # type: (List[str], logging.Logger) -> None cnt = 0 for queue_name in queues: if not settings.DEVELOPMENT: logger.info('launching queue worker thread ' + queue_name) cnt += 1 td = Threaded_worker(queue_name) td.start() assert len(queues) == cnt logger.info('%d queue worker threads were launched' % (cnt,)) if options['all']: signal.signal(signal.SIGUSR1, exit_with_three) autoreload.main(run_threaded_workers, (get_active_worker_queues(), logger)) elif options['multi_threaded']: signal.signal(signal.SIGUSR1, exit_with_three) queues = options['multi_threaded'] autoreload.main(run_threaded_workers, (queues, logger)) else: queue_name = options['queue_name'] worker_num = options['worker_num'] logger.info("Worker %d connecting to queue %s" % (worker_num, queue_name)) worker = get_worker(queue_name) worker.setup() def signal_handler(signal, frame): # type: (int, FrameType) -> None logger.info("Worker %d disconnecting from queue %s" % (worker_num, queue_name)) worker.stop() sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGUSR1, signal_handler) worker.start()
def handle(self, *args: Any, **options: Any) -> None: logging.basicConfig() logger = logging.getLogger('process_queue') def exit_with_three(signal: int, frame: FrameType) -> None: """ This process is watched by Django's autoreload, so exiting with status code 3 will cause this process to restart. """ logger.warning("SIGUSR1 received. Restarting this queue processor.") sys.exit(3) if not settings.USING_RABBITMQ: # Make the warning silent when running the tests if settings.TEST_SUITE: logger.info("Not using RabbitMQ queue workers in the test suite.") else: logger.error("Cannot run a queue processor when USING_RABBITMQ is False!") raise CommandError def run_threaded_workers(queues: List[str], logger: logging.Logger) -> None: cnt = 0 for queue_name in queues: if not settings.DEVELOPMENT: logger.info('launching queue worker thread ' + queue_name) cnt += 1 td = Threaded_worker(queue_name) td.start() assert len(queues) == cnt logger.info('%d queue worker threads were launched' % (cnt,)) if options['all']: signal.signal(signal.SIGUSR1, exit_with_three) autoreload.main(run_threaded_workers, (get_active_worker_queues(), logger)) elif options['multi_threaded']: signal.signal(signal.SIGUSR1, exit_with_three) queues = options['multi_threaded'] autoreload.main(run_threaded_workers, (queues, logger)) else: queue_name = options['queue_name'] worker_num = options['worker_num'] logger.info("Worker %d connecting to queue %s" % (worker_num, queue_name)) worker = get_worker(queue_name) worker.setup() def signal_handler(signal: int, frame: FrameType) -> None: logger.info("Worker %d disconnecting from queue %s" % (worker_num, queue_name)) worker.stop() sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGUSR1, signal_handler) worker.start()
def handle(self, *args, **options): # type: (*Any, **Any) -> None logging.basicConfig() logger = logging.getLogger('process_queue') if not settings.USING_RABBITMQ: # Make the warning silent when running the tests if settings.TEST_SUITE: logger.info( "Not using RabbitMQ queue workers in the test suite.") else: logger.error( "Cannot run a queue processor when USING_RABBITMQ is False!" ) sys.exit(1) def run_threaded_workers(logger): # type: (logging.Logger) -> None cnt = 0 for queue_name in get_active_worker_queues(): if not settings.DEVELOPMENT: logger.info('launching queue worker thread ' + queue_name) cnt += 1 td = Threaded_worker(queue_name) td.start() logger.info('%d queue worker threads were launched' % (cnt, )) if options['all']: autoreload.main(run_threaded_workers, (logger, )) else: queue_name = options['queue_name'] worker_num = options['worker_num'] logger.info("Worker %d connecting to queue %s" % (worker_num, queue_name)) worker = get_worker(queue_name) worker.setup() def signal_handler(signal, frame): # type: (int, FrameType) -> None logger.info("Worker %d disconnecting from queue %s" % (worker_num, queue_name)) worker.stop() sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) worker.start()
def handle(self, *args, **options): # type: (*Any, **Any) -> None logging.basicConfig() logger = logging.getLogger('process_queue') if not settings.USING_RABBITMQ: # Make the warning silent when running the tests if settings.TEST_SUITE: logger.info("Not using RabbitMQ queue workers in the test suite.") else: logger.error("Cannot run a queue processor when USING_RABBITMQ is False!") sys.exit(1) def run_threaded_workers(logger): # type: (logging.Logger) -> None cnt = 0 for queue_name in get_active_worker_queues(): if not settings.DEVELOPMENT: logger.info('launching queue worker thread ' + queue_name) cnt += 1 td = Threaded_worker(queue_name) td.start() logger.info('%d queue worker threads were launched' % (cnt,)) if options['all']: autoreload.main(run_threaded_workers, (logger,)) else: queue_name = options['queue_name'] worker_num = options['worker_num'] logger.info("Worker %d connecting to queue %s" % (worker_num, queue_name)) worker = get_worker(queue_name) worker.setup() def signal_handler(signal, frame): # type: (int, FrameType) -> None logger.info("Worker %d disconnecting from queue %s" % (worker_num, queue_name)) worker.stop() sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) worker.start()
def handle(self, *args, **options): logging.basicConfig() logger = logging.getLogger('process_queue') queue_name = options['queue_name'] worker_num = options['worker_num'] def signal_handler(signal, frame): logger.info("Worker %d disconnecting from queue %s" % (worker_num, queue_name)) worker.stop() sys.exit(0) if not settings.USING_RABBITMQ: logger.error("Cannot run a queue processor when USING_RABBITMQ is False!") sys.exit(1) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) logger.info("Worker %d connecting to queue %s" % (worker_num, queue_name)) worker = get_worker(queue_name) worker.start()
def __init__(self, queue_name): # type: (str) -> None threading.Thread.__init__(self) self.worker = get_worker(queue_name)
def __init__(self, queue_name): threading.Thread.__init__(self) self.worker = get_worker(queue_name)