def handle(self, *args, **options): queue = options["queue"][0] concurrency = options["concurrency"] or 1 print("Starting celery worker with autoreload...") autoreload.run_with_reloader(restart_celery, queue=queue, concurrency=concurrency)
def handle(self, *args, **options): print("Starting celery worker with autoreload...") # noqa try: os.remove("celerybeat.pid") except FileNotFoundError as e: print(e) # noqa autoreload.run_with_reloader(restart_celery_beat)
def test_calls_start_django(self, mocked_reloader, mocked_start_django): mocked_reloader.return_value = mock.sentinel.RELOADER autoreload.run_with_reloader(mock.sentinel.METHOD) self.assertEqual(mocked_start_django.call_count, 1) self.assertSequenceEqual( mocked_start_django.call_args[0], [mock.sentinel.RELOADER, mock.sentinel.METHOD])
def handle(self, *args, celery_type, **options): # pylint: disable=arguments-differ if not settings.DEBUG: raise CommandError( "This command can only be ran when DEBUG is True") self.stdout.write(f"Starting celery {celery_type} with autoreload...") autoreload.run_with_reloader(restart_celery, **{"celery_type": celery_type})
def handle(self, *args, **options): use_reloader = options['use_reloader'] if use_reloader: autoreload.run_with_reloader(self.run) else: self.run()
def handle(self, *args, **options): queue = options['queue'][0] concurrency = options['concurrency'] or 1 print('Starting celery worker with autoreload...') autoreload.run_with_reloader(restart_celery, queue=queue, concurrency=concurrency)
def handle(self, *args: Any, **options: Any) -> None: logging.basicConfig() logger = logging.getLogger("process_queue") def exit_with_three(signal: int, frame: Optional[FrameType]) -> None: """ This process is watched by Django's autoreload, so exiting with status code 3 will cause this process to restart. """ logger.warning("SIGUSR1 received. Restarting this queue processor.") sys.exit(3) if not settings.USING_RABBITMQ: # Make the warning silent when running the tests if settings.TEST_SUITE: logger.info("Not using RabbitMQ queue workers in the test suite.") else: logger.error("Cannot run a queue processor when USING_RABBITMQ is False!") raise CommandError def run_threaded_workers(queues: List[str], logger: logging.Logger) -> None: cnt = 0 for queue_name in queues: if not settings.DEVELOPMENT: logger.info("launching queue worker thread %s", queue_name) cnt += 1 td = ThreadedWorker(queue_name, logger) td.start() assert len(queues) == cnt logger.info("%d queue worker threads were launched", cnt) if options["all"]: signal.signal(signal.SIGUSR1, exit_with_three) autoreload.run_with_reloader(run_threaded_workers, get_active_worker_queues(), logger) elif options["multi_threaded"]: signal.signal(signal.SIGUSR1, exit_with_three) queues = options["multi_threaded"] autoreload.run_with_reloader(run_threaded_workers, queues, logger) else: queue_name = options["queue_name"] worker_num = options["worker_num"] def signal_handler(signal: int, frame: Optional[FrameType]) -> None: logger.info("Worker %d disconnecting from queue %s", worker_num, queue_name) worker.stop() sys.exit(0) logger.info("Worker %d connecting to queue %s", worker_num, queue_name) with log_and_exit_if_exception(logger, queue_name, threaded=False): worker = get_worker(queue_name) with configure_scope() as scope: scope.set_tag("queue_worker", queue_name) scope.set_tag("worker_num", worker_num) worker.setup() signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGUSR1, signal_handler) worker.ENABLE_TIMEOUTS = True worker.start()
def handle(self, *args, **options): options_schema = options.get("schema") if options_schema and type(options_schema) is str: module_str, schema_name = options_schema.rsplit(".", 1) mod = importlib.import_module(module_str) schema = getattr(mod, schema_name) elif options_schema: schema = options_schema else: schema = graphene_settings.SCHEMA out = options.get("out") or graphene_settings.SCHEMA_OUTPUT if not schema: raise CommandError( "Specify schema on GRAPHENE.SCHEMA setting or by using --schema" ) indent = options.get("indent") watch = options.get("watch") if watch: autoreload.run_with_reloader( functools.partial(self.get_schema, schema, out, indent)) else: self.get_schema(schema, out, indent)
def worker(ignore_unknown_queues, **options): """Run background worker instance and autoreload if necessary.""" from sentry.celery import app known_queues = frozenset(c_queue.name for c_queue in app.conf.CELERY_QUEUES) if options["queues"] is not None: if not options["queues"].issubset(known_queues): unknown_queues = options["queues"] - known_queues message = "Following queues are not found: %s" % ",".join(sorted(unknown_queues)) if ignore_unknown_queues: options["queues"] -= unknown_queues click.echo(message) else: raise click.ClickException(message) if options["exclude_queues"] is not None: if not options["exclude_queues"].issubset(known_queues): unknown_queues = options["exclude_queues"] - known_queues message = "Following queues cannot be excluded as they don't exist: %s" % ",".join( sorted(unknown_queues) ) if ignore_unknown_queues: options["exclude_queues"] -= unknown_queues click.echo(message) else: raise click.ClickException(message) if options["autoreload"]: from django.utils import autoreload autoreload.run_with_reloader(run_worker, kwargs=options) else: run_worker(**options)
def handle(self, *args, **options): if options["autoreload"] is True: self.stdout.write("ATTENTION! Autoreload is enabled!") # autoreload.main(self._serve, None, options) autoreload.run_with_reloader(self._serve, **options) else: self._serve(**options)
def run(self, **options): """Run the server, using the autoreloader if needed.""" use_reloader = options['use_reloader'] if use_reloader: autoreload.run_with_reloader(self.inner_run, **options) else: self.inner_run(None, **options)
def handle(self, *args, **options): print('Starting celery worker with autoreload...') # For Django>=2.2 autoreload.run_with_reloader(restart_celery) # For django<2.1 # autoreload.main(restart_celery)
def test_calls_start_django(self, mocked_reloader, mocked_start_django): mocked_reloader.return_value = mock.sentinel.RELOADER autoreload.run_with_reloader(mock.sentinel.METHOD) self.assertEqual(mocked_start_django.call_count, 1) self.assertSequenceEqual( mocked_start_django.call_args[0], [mock.sentinel.RELOADER, mock.sentinel.METHOD] )
def run(self, **options): """Run the server, using the autoreloader if needed.""" use_reloader = options['use_reloader'] if use_reloader: autoreload.run_with_reloader(self.inner_run, **options) else: self.inner_run(None, **options)
def handle(self, *args, **options): self.stdout.write('Starting celery worker with autoreload...') kwargs = { 'loglevel': options.get('loglevel'), 'logfile': options.get('logfile'), 'uid': options.get('uid'), 'gid': options.get('gid'), } autoreload.run_with_reloader(restart_celery, args=None, kwargs=kwargs)
def handle(self, *args, **options): print('Starting celery workers with autoreload...') try: from django.utils.autoreload import run_with_reloader run_with_reloader(restart_celery) except ImportError: from django.utils import autoreload autoreload.main(restart_celery)
def handle(self, *args, **options): updater = Updater(settings.TELEGRAM_TOKEN) updater.dispatcher.groups = [DEFAULT_GROUP] updater.dispatcher.handlers[DEFAULT_GROUP] = CALLBACKS if settings.DEBUG: autoreload.run_with_reloader(updater.start_polling) else: updater.start_polling() updater.idle()
def run(self, **options): # self 就是「命令处理对象」 # 下面这个变量的默认值是 True ,如果有代码变动,会自动重新启动程序 use_reloader = options['use_reloader'] if use_reloader: # 此函数定义在 django.utils.autoreload 模块中 autoreload.run_with_reloader(self.inner_run, **options) else: self.inner_run(None, **options)
def run_script(self): from django.conf import settings from celery.bin.celery import main as celery_main if settings.DEBUG and "worker" in sys.argv and "-h" not in sys.argv: if callable(run_with_reloader): run_with_reloader(celery_main, sys.argv) elif callable(python_reloader): python_reloader(celery_main, (sys.argv, ), {}) else: celery_main(sys.argv)
def run(self, **options): # 运行服务器,如果需要使用自动加载程序。." """Run the server, using the autoreloader if needed.""" use_reloader = options['use_reloader'] # True if use_reloader: # inner_run 为 Django 开启的主线程的 目标回调函数 # autoreload = django\utils\autoreload.py autoreload.run_with_reloader(self.inner_run, **options) # 关键代码 else: self.inner_run(None, **options)
def handle(self, *args, **options): if options['autoreload'] is True: self.stdout.write("ATTENTION! Autoreload is enabled!") if hasattr(autoreload, "run_with_reloader"): # Django 2.2. and above autoreload.run_with_reloader(self._serve, **options) else: # Before Django 2.2. autoreload.main(self._serve, None, options) else: self._serve(**options)
def handle(self, *args, **options): """Start Celery""" def restart_celery(): """Restart celery worker""" print('Starting celery worker with autoreload...') cmd = 'pkill -9 celery' subprocess.call(shlex.split(cmd)) cmd = 'celery worker -A {} -l info'.format( options.get('celery_app')) subprocess.call(shlex.split(cmd)) autoreload.run_with_reloader(restart_celery)
def handle(self, *args, **options): if options.get('use_reloader'): self.stdout.write('Starting celery with autoreload...') run_with_reloader(start_celery, celery_type=options.get('type'), celery_settings=options.get('celery_settings'), extra_arguments=options.get('extra_args', ''), autoreload=True) else: self.stdout.write('Starting celery...') start_celery(celery_type=options.get('type'), celery_settings=options.get('celery_settings'), extra_arguments=options.get('extra_args', ''))
def handle(self, *args, **options): print('Starting celery worker with autoreload...') try: #restart_celery(args=None, kwargs=None) autoreload.run_with_reloader(restart_celery, args=None, kwargs=None) except CommandError: content = [{"title": "celery.py", "text": "celery reload failed"}] opererror_slack(attachments=content) self.stdout.write(self.style.ERROR('Celery reload failed')) else: self.stdout.write( self.style.SUCCESS('Successfully celery reload '))
def handle(self, *args, **options): broker_url = options['broker_url'] log_level = options['log_level'] queues = options['queues'] args = (broker_url, log_level, queues) # close over args def restart_celery_worker(): run_cmd('pkill -f "celery worker"') run_cmd('celery worker -A config.celery -b %s ' '--loglevel=%s -Q %s' % args) autoreload.run_with_reloader(restart_celery_worker)
def run(self, **options): """Run the server, using the autoreloader if needed.""" if self.development_mode: if hasattr(autoreload, "run_with_reloader"): autoreload.run_with_reloader(self.inner_run, **options) else: autoreload.main(self.inner_run, None, options) else: self.stdout.write(( "Starting gRPC server at %(address)s\n" ) % { "address": self.address, }) self._serve()
async def run(self, **options): """Run the server, using the autoreloader if needed.""" if self.development_mode: if hasattr(autoreload, "run_with_reloader"): autoreload.run_with_reloader(self.inner_run, **options) else: autoreload.main(self.inner_run, None, options) else: logger.info( ("Starting async gRPC server at %(address)s\n") % { "address": self.address, }, extra={"emit_to_server": False}, ) await self._serve()
def run_from_argv(self, argv): os.environ.setdefault("CELERY_APP", ws_settings.CELERY_APP) if settings.DEBUG and "-h" not in sys.argv: if not any(x in sys.argv for x in ("-c", "--concurrency", "-P", "--pool")): sys.argv += ["-c", "1", "--pool", "solo"] return run_with_reloader(celery_main) celery_main()
def handle(self, *args, **kwargs): print("Starting celery worker with autoreload") run_with_reloader(restart_celery)
def test_calls_sys_exit(self, mocked_restart_reloader): mocked_restart_reloader.return_value = 1 with self.assertRaises(SystemExit) as exc: autoreload.run_with_reloader(lambda: None) self.assertEqual(exc.exception.code, 1)
def test_swallows_keyboard_interrupt(self, mocked_get_reloader): mocked_get_reloader.side_effect = KeyboardInterrupt() autoreload.run_with_reloader(lambda: None) # No exception
def handle(self, *args, **options): autoreload.run_with_reloader(restart_celery_worker)
def test_calls_sys_exit(self, mocked_restart_reloader): mocked_restart_reloader.return_value = 1 with self.assertRaises(SystemExit) as exc: autoreload.run_with_reloader(lambda: None) self.assertEqual(exc.exception.code, 1)
def test_swallows_keyboard_interrupt(self, mocked_get_reloader): mocked_get_reloader.side_effect = KeyboardInterrupt() autoreload.run_with_reloader(lambda: None) # No exception