def handle(self, *args, **options):
queue = options["queue"][0]
concurrency = options["concurrency"] or 1
print("Starting celery worker with autoreload...")
autoreload.run_with_reloader(restart_celery,
queue=queue,
concurrency=concurrency)
def handle(self, *args, **options):
print("Starting celery worker with autoreload...") # noqa
try:
os.remove("celerybeat.pid")
except FileNotFoundError as e:
print(e) # noqa
autoreload.run_with_reloader(restart_celery_beat)
def test_calls_start_django(self, mocked_reloader, mocked_start_django):
mocked_reloader.return_value = mock.sentinel.RELOADER
autoreload.run_with_reloader(mock.sentinel.METHOD)
self.assertEqual(mocked_start_django.call_count, 1)
self.assertSequenceEqual(
mocked_start_django.call_args[0],
[mock.sentinel.RELOADER, mock.sentinel.METHOD])
def handle(self, *args, celery_type, **options): # pylint: disable=arguments-differ
if not settings.DEBUG:
raise CommandError(
"This command can only be ran when DEBUG is True")
self.stdout.write(f"Starting celery {celery_type} with autoreload...")
autoreload.run_with_reloader(restart_celery,
**{"celery_type": celery_type})
def handle(self, *args, **options):
use_reloader = options['use_reloader']
if use_reloader:
autoreload.run_with_reloader(self.run)
else:
self.run()
def handle(self, *args, **options):
queue = options['queue'][0]
concurrency = options['concurrency'] or 1
print('Starting celery worker with autoreload...')
autoreload.run_with_reloader(restart_celery,
queue=queue,
concurrency=concurrency)
def handle(self, *args: Any, **options: Any) -> None:
logging.basicConfig()
logger = logging.getLogger("process_queue")
def exit_with_three(signal: int, frame: Optional[FrameType]) -> None:
"""
This process is watched by Django's autoreload, so exiting
with status code 3 will cause this process to restart.
"""
logger.warning("SIGUSR1 received. Restarting this queue processor.")
sys.exit(3)
if not settings.USING_RABBITMQ:
# Make the warning silent when running the tests
if settings.TEST_SUITE:
logger.info("Not using RabbitMQ queue workers in the test suite.")
else:
logger.error("Cannot run a queue processor when USING_RABBITMQ is False!")
raise CommandError
def run_threaded_workers(queues: List[str], logger: logging.Logger) -> None:
cnt = 0
for queue_name in queues:
if not settings.DEVELOPMENT:
logger.info("launching queue worker thread %s", queue_name)
cnt += 1
td = ThreadedWorker(queue_name, logger)
td.start()
assert len(queues) == cnt
logger.info("%d queue worker threads were launched", cnt)
if options["all"]:
signal.signal(signal.SIGUSR1, exit_with_three)
autoreload.run_with_reloader(run_threaded_workers, get_active_worker_queues(), logger)
elif options["multi_threaded"]:
signal.signal(signal.SIGUSR1, exit_with_three)
queues = options["multi_threaded"]
autoreload.run_with_reloader(run_threaded_workers, queues, logger)
else:
queue_name = options["queue_name"]
worker_num = options["worker_num"]
def signal_handler(signal: int, frame: Optional[FrameType]) -> None:
logger.info("Worker %d disconnecting from queue %s", worker_num, queue_name)
worker.stop()
sys.exit(0)
logger.info("Worker %d connecting to queue %s", worker_num, queue_name)
with log_and_exit_if_exception(logger, queue_name, threaded=False):
worker = get_worker(queue_name)
with configure_scope() as scope:
scope.set_tag("queue_worker", queue_name)
scope.set_tag("worker_num", worker_num)
worker.setup()
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGUSR1, signal_handler)
worker.ENABLE_TIMEOUTS = True
worker.start()
def handle(self, *args, **options):
options_schema = options.get("schema")
if options_schema and type(options_schema) is str:
module_str, schema_name = options_schema.rsplit(".", 1)
mod = importlib.import_module(module_str)
schema = getattr(mod, schema_name)
elif options_schema:
schema = options_schema
else:
schema = graphene_settings.SCHEMA
out = options.get("out") or graphene_settings.SCHEMA_OUTPUT
if not schema:
raise CommandError(
"Specify schema on GRAPHENE.SCHEMA setting or by using --schema"
)
indent = options.get("indent")
watch = options.get("watch")
if watch:
autoreload.run_with_reloader(
functools.partial(self.get_schema, schema, out, indent))
else:
self.get_schema(schema, out, indent)
def worker(ignore_unknown_queues, **options):
"""Run background worker instance and autoreload if necessary."""
from sentry.celery import app
known_queues = frozenset(c_queue.name for c_queue in app.conf.CELERY_QUEUES)
if options["queues"] is not None:
if not options["queues"].issubset(known_queues):
unknown_queues = options["queues"] - known_queues
message = "Following queues are not found: %s" % ",".join(sorted(unknown_queues))
if ignore_unknown_queues:
options["queues"] -= unknown_queues
click.echo(message)
else:
raise click.ClickException(message)
if options["exclude_queues"] is not None:
if not options["exclude_queues"].issubset(known_queues):
unknown_queues = options["exclude_queues"] - known_queues
message = "Following queues cannot be excluded as they don't exist: %s" % ",".join(
sorted(unknown_queues)
)
if ignore_unknown_queues:
options["exclude_queues"] -= unknown_queues
click.echo(message)
else:
raise click.ClickException(message)
if options["autoreload"]:
from django.utils import autoreload
autoreload.run_with_reloader(run_worker, kwargs=options)
else:
run_worker(**options)
def handle(self, *args, **options):
if options["autoreload"] is True:
self.stdout.write("ATTENTION! Autoreload is enabled!")
# autoreload.main(self._serve, None, options)
autoreload.run_with_reloader(self._serve, **options)
else:
self._serve(**options)
def run(self, **options):
"""Run the server, using the autoreloader if needed."""
use_reloader = options['use_reloader']
if use_reloader:
autoreload.run_with_reloader(self.inner_run, **options)
else:
self.inner_run(None, **options)
def handle(self, *args, **options):
print('Starting celery worker with autoreload...')
# For Django>=2.2
autoreload.run_with_reloader(restart_celery)
# For django<2.1
# autoreload.main(restart_celery)
def test_calls_start_django(self, mocked_reloader, mocked_start_django):
mocked_reloader.return_value = mock.sentinel.RELOADER
autoreload.run_with_reloader(mock.sentinel.METHOD)
self.assertEqual(mocked_start_django.call_count, 1)
self.assertSequenceEqual(
mocked_start_django.call_args[0],
[mock.sentinel.RELOADER, mock.sentinel.METHOD]
)
def run(self, **options):
"""Run the server, using the autoreloader if needed."""
use_reloader = options['use_reloader']
if use_reloader:
autoreload.run_with_reloader(self.inner_run, **options)
else:
self.inner_run(None, **options)
def handle(self, *args, **options):
self.stdout.write('Starting celery worker with autoreload...')
kwargs = {
'loglevel': options.get('loglevel'),
'logfile': options.get('logfile'),
'uid': options.get('uid'),
'gid': options.get('gid'),
}
autoreload.run_with_reloader(restart_celery, args=None, kwargs=kwargs)
def handle(self, *args, **options):
print('Starting celery workers with autoreload...')
try:
from django.utils.autoreload import run_with_reloader
run_with_reloader(restart_celery)
except ImportError:
from django.utils import autoreload
autoreload.main(restart_celery)
def handle(self, *args, **options):
updater = Updater(settings.TELEGRAM_TOKEN)
updater.dispatcher.groups = [DEFAULT_GROUP]
updater.dispatcher.handlers[DEFAULT_GROUP] = CALLBACKS
if settings.DEBUG:
autoreload.run_with_reloader(updater.start_polling)
else:
updater.start_polling()
updater.idle()
def run(self, **options):
# self 就是「命令处理对象」
# 下面这个变量的默认值是 True ，如果有代码变动，会自动重新启动程序
use_reloader = options['use_reloader']
if use_reloader:
# 此函数定义在 django.utils.autoreload 模块中
autoreload.run_with_reloader(self.inner_run, **options)
else:
self.inner_run(None, **options)
def run_script(self):
from django.conf import settings
from celery.bin.celery import main as celery_main
if settings.DEBUG and "worker" in sys.argv and "-h" not in sys.argv:
if callable(run_with_reloader):
run_with_reloader(celery_main, sys.argv)
elif callable(python_reloader):
python_reloader(celery_main, (sys.argv, ), {})
else:
celery_main(sys.argv)
def run(self, **options):
# 运行服务器，如果需要使用自动加载程序。."
"""Run the server, using the autoreloader if needed."""
use_reloader = options['use_reloader'] # True
if use_reloader:
# inner_run 为 Django 开启的主线程的 目标回调函数
# autoreload = django\utils\autoreload.py
autoreload.run_with_reloader(self.inner_run, **options) # 关键代码
else:
self.inner_run(None, **options)
def handle(self, *args, **options):
if options['autoreload'] is True:
self.stdout.write("ATTENTION! Autoreload is enabled!")
if hasattr(autoreload, "run_with_reloader"):
# Django 2.2. and above
autoreload.run_with_reloader(self._serve, **options)
else:
# Before Django 2.2.
autoreload.main(self._serve, None, options)
else:
self._serve(**options)
def handle(self, *args, **options):
"""Start Celery"""
def restart_celery():
"""Restart celery worker"""
print('Starting celery worker with autoreload...')
cmd = 'pkill -9 celery'
subprocess.call(shlex.split(cmd))
cmd = 'celery worker -A {} -l info'.format(
options.get('celery_app'))
subprocess.call(shlex.split(cmd))
autoreload.run_with_reloader(restart_celery)
def handle(self, *args, **options):
if options.get('use_reloader'):
self.stdout.write('Starting celery with autoreload...')
run_with_reloader(start_celery,
celery_type=options.get('type'),
celery_settings=options.get('celery_settings'),
extra_arguments=options.get('extra_args', ''),
autoreload=True)
else:
self.stdout.write('Starting celery...')
start_celery(celery_type=options.get('type'),
celery_settings=options.get('celery_settings'),
extra_arguments=options.get('extra_args', ''))
def handle(self, *args, **options):
print('Starting celery worker with autoreload...')
try:
#restart_celery(args=None, kwargs=None)
autoreload.run_with_reloader(restart_celery,
args=None,
kwargs=None)
except CommandError:
content = [{"title": "celery.py", "text": "celery reload failed"}]
opererror_slack(attachments=content)
self.stdout.write(self.style.ERROR('Celery reload failed'))
else:
self.stdout.write(
self.style.SUCCESS('Successfully celery reload '))
def handle(self, *args, **options):
broker_url = options['broker_url']
log_level = options['log_level']
queues = options['queues']
args = (broker_url, log_level, queues)
# close over args
def restart_celery_worker():
run_cmd('pkill -f "celery worker"')
run_cmd('celery worker -A config.celery -b %s '
'--loglevel=%s -Q %s' % args)
autoreload.run_with_reloader(restart_celery_worker)
def run(self, **options):
"""Run the server, using the autoreloader if needed."""
if self.development_mode:
if hasattr(autoreload, "run_with_reloader"):
autoreload.run_with_reloader(self.inner_run, **options)
else:
autoreload.main(self.inner_run, None, options)
else:
self.stdout.write((
"Starting gRPC server at %(address)s\n"
) % {
"address": self.address,
})
self._serve()
async def run(self, **options):
"""Run the server, using the autoreloader if needed."""
if self.development_mode:
if hasattr(autoreload, "run_with_reloader"):
autoreload.run_with_reloader(self.inner_run, **options)
else:
autoreload.main(self.inner_run, None, options)
else:
logger.info(
("Starting async gRPC server at %(address)s\n") % {
"address": self.address,
},
extra={"emit_to_server": False},
)
await self._serve()
def run_from_argv(self, argv):
os.environ.setdefault("CELERY_APP", ws_settings.CELERY_APP)
if settings.DEBUG and "-h" not in sys.argv:
if not any(x in sys.argv
for x in ("-c", "--concurrency", "-P", "--pool")):
sys.argv += ["-c", "1", "--pool", "solo"]
return run_with_reloader(celery_main)
celery_main()
def handle(self, *args, **kwargs):
print("Starting celery worker with autoreload")
run_with_reloader(restart_celery)
def test_calls_sys_exit(self, mocked_restart_reloader):
mocked_restart_reloader.return_value = 1
with self.assertRaises(SystemExit) as exc:
autoreload.run_with_reloader(lambda: None)
self.assertEqual(exc.exception.code, 1)
def test_swallows_keyboard_interrupt(self, mocked_get_reloader):
mocked_get_reloader.side_effect = KeyboardInterrupt()
autoreload.run_with_reloader(lambda: None) # No exception
def handle(self, *args, **options):
autoreload.run_with_reloader(restart_celery_worker)
def test_calls_sys_exit(self, mocked_restart_reloader):
mocked_restart_reloader.return_value = 1
with self.assertRaises(SystemExit) as exc:
autoreload.run_with_reloader(lambda: None)
self.assertEqual(exc.exception.code, 1)
def test_swallows_keyboard_interrupt(self, mocked_get_reloader):
mocked_get_reloader.side_effect = KeyboardInterrupt()
autoreload.run_with_reloader(lambda: None) # No exception
