def main() -> None:
"""
Command-line parser. See command-line help.
"""
# noinspection PyTypeChecker
parser = argparse.ArgumentParser(
description="Launch CRATE NLP web server Celery processes. "
"(Any leftover arguments will be passed to Celery.)",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--command", default="worker", help="Celery command")
parser.add_argument(
"--cleanup_timeout_s",
type=float,
default=10.0,
help="Time to wait when shutting down Celery via Ctrl-C")
parser.add_argument("--debug",
action="store_true",
help="Ask Celery to be verbose")
args, leftovers = parser.parse_known_args()
cmdargs = [
"celery",
args.command,
"-A",
NLP_WEBSERVER_CELERY_APP_NAME,
]
if args.command == "worker":
cmdargs += ["-l", "debug" if args.debug else "info"] # --loglevel
if WINDOWS:
# See launch_celery.py
cmdargs += ["--concurrency=1"]
cmdargs += ["--pool=solo"]
cmdargs += leftovers
print(f"Launching Celery: {cmdargs}")
nice_call(cmdargs, cleanup_timeout=args.cleanup_timeout_s)
def launch_celery_beat(
verbose: bool = False, cleanup_timeout_s: float = DEFAULT_CLEANUP_TIMEOUT_S
) -> None:
"""
Launch the Celery Beat scheduler.
(This can be combined with ``celery worker``, but that's not recommended;
https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#starting-the-scheduler).
""" # noqa: E501
ensure_directories_exist()
config = get_default_config_from_os_env()
cmdargs = [
CELERY,
"--app",
CELERY_APP_NAME,
"beat",
"--schedule",
config.celery_beat_schedule_database,
"--pidfile",
config.get_celery_beat_pidfilename(),
"--loglevel",
"DEBUG" if verbose else "INFO",
]
cmdargs += config.celery_beat_extra_args
log.info("Launching: {!r}", cmdargs)
nice_call(cmdargs, cleanup_timeout=cleanup_timeout_s)
def launch_celery_workers(
verbose: bool = False, cleanup_timeout_s: float = DEFAULT_CLEANUP_TIMEOUT_S
) -> None:
"""
Launch Celery workers.
See also advice in
- https://medium.com/@taylorhughes/three-quick-tips-from-two-years-with-celery-c05ff9d7f9eb
- Re ``-Ofair``:
https://docs.celeryproject.org/en/latest/userguide/optimizing.html
""" # noqa: E501
config = get_default_config_from_os_env()
cmdargs = [
CELERY,
"--app",
CELERY_APP_NAME,
"worker",
"-O",
"fair", # optimization
"--soft-time-limit",
str(CELERY_SOFT_TIME_LIMIT_SEC),
"--loglevel",
"DEBUG" if verbose else "INFO",
]
if WINDOWS:
# See crate_anon/tools/launch_celery.py, and
# camcops_server/cc_modules/cc_export.py
os.environ["FORKED_BY_MULTIPROCESSING"] = "1"
cmdargs.extend(["--concurrency", "1", "--pool", "solo"])
cmdargs += config.celery_worker_extra_args
log.info("Launching: {!r}", cmdargs)
nice_call(cmdargs, cleanup_timeout=cleanup_timeout_s)
def inner_main() -> None:
"""
Command-line parser. See command-line help.
"""
# noinspection PyTypeChecker
parser = argparse.ArgumentParser(
description="Launch CRATE Celery processes. "
"(Any leftover arguments will be passed to Celery.)",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--command", default="worker", help="Celery command")
parser.add_argument(
"--cleanup_timeout_s",
type=float,
default=10.0,
help="Time to wait when shutting down Celery via Ctrl-C")
parser.add_argument("--debug",
action="store_true",
help="Ask Celery to be verbose")
args, leftovers = parser.parse_known_args()
# print(f"Changing to directory: {DJANGO_ROOT}")
# os.chdir(DJANGO_ROOT)
cmdargs = [
"celery",
args.command,
"-A",
CRATEWEB_CELERY_APP_NAME,
]
if args.command == "worker":
cmdargs += ["-l", "debug" if args.debug else "info"] # --loglevel
if WINDOWS:
# Default concurrency is 1. Under Celery 3.1.23, RabbitMQ 3.6.1,
# and Windows 10, things like "celery -A myapp status" don't work
# unless the concurrency flag is increased, e.g. to 4.
# (Default is 1 under Windows.)
# cmdargs += ["--concurrency=4"]
cmdargs += ["--concurrency=1"] # 2018-06-29
# Without "--pool=solo", sod all happens: tasks go into the
# Reserved queue, but aren't executed.
# See:
# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#module-celery.bin.worker # noqa
# https://github.com/celery/celery/issues/2146
cmdargs += ["--pool=solo"]
# The "-Ofair" option is relevant to pre-fetching, but doesn't
# seem critical.
# cmdargs += ["-Ofair"]
# We don't need to specify modules manually, now we have a package.
# modules = get_python_modules(DJANGO_ROOT,
# prefix="crate_anon.crateweb.")
# cmdargs += ["--include", f"{','.join(modules)}"]
# "--autoreload",
cmdargs += leftovers
print(f"Launching Celery: {cmdargs}")
nice_call(cmdargs, cleanup_timeout=args.cleanup_timeout_s)
def launch_celery_flower(
address: str = DEFAULT_FLOWER_ADDRESS,
port: int = DEFAULT_FLOWER_PORT,
cleanup_timeout_s: float = DEFAULT_CLEANUP_TIMEOUT_S,
) -> None:
"""
Launch the Celery Flower monitor.
"""
cmdargs = [
CELERY,
"--app",
CELERY_APP_NAME,
"flower",
f"--address {address}",
f"--port {port}",
]
log.info("Launching: {!r}", cmdargs)
nice_call(cmdargs, cleanup_timeout=cleanup_timeout_s)