def _connect_signals(self):
"""Connect callbacks to celery signals.
Since we are creating partials here, the weak arg must be False.
"""
# Beat
if self.options["show_beat"]:
beat_init.connect(
slack_beat_init(**self.options),
weak=False
)
# Celery
if self.options["show_startup"]:
celeryd_init.connect(
slack_celery_startup(**self.options),
weak=False
)
if self.options["show_shutdown"]:
worker_shutdown.connect(
slack_celery_shutdown(**self.options),
weak=False
)
# Task
task_prerun.connect(
slack_task_prerun(**self.options),
weak=False
)
def init_app(self, app):
self.app = app
self.app_ctx = app.app_context()
# base_url = self._parse_app_url(app)
# self.req_ctx = app.test_request_context(base_url=base_url)
self.req_ctx = app.test_request_context()
new_celery = celery.Celery(
app.import_name,
broker=app.config["CELERY_BROKER_URL"],
backend=app.config["CELERY_RESULT_BACKEND"],
enable_utc=True,
timezone=app.config["BABEL_DEFAULT_TIMEZONE"],
)
self.celery.__dict__.update(vars(new_celery))
self.celery.conf.update(app.config)
self.celery.conf["BROKER_HEARTBEAT"] = 0
celeryd_init.connect(self._worker_process_init)
task_postrun.connect(self._task_postrun)
task_prerun.connect(self._task_prerun)
# SPDX-License-Identifier: GPL-3.0-or-later
import sys
import celery
from celery.signals import celeryd_init, task_postrun, task_prerun
from cachito.workers.celery_logging import (
cleanup_task_logging,
cleanup_task_logging_customization,
setup_task_logging,
setup_task_logging_customization,
)
from cachito.workers.config import configure_celery, validate_celery_config
# Workaround https://github.com/celery/celery/issues/5416
if celery.version_info < (4, 3) and sys.version_info >= (
3, 7): # pragma: no cover
from re import Pattern
from celery.app.routes import re as routes_re
routes_re._pattern_type = Pattern
app = celery.Celery()
configure_celery(app)
celeryd_init.connect(validate_celery_config)
task_prerun.connect(setup_task_logging_customization)
task_prerun.connect(setup_task_logging)
task_postrun.connect(cleanup_task_logging_customization)
task_postrun.connect(cleanup_task_logging)
# this works but I think this way is legacy and has problems shutting down
# gracefully. Will be reimplemented using a custom `bootstep`
# tunnel.create_server(
# redis_server,
# local_node_port,
# ssh_password=ssh_password,
# # ssh_host_key=ssh_host_key
# )
tun = tunnel.BashTunnel(
redis_server,
local_node_port
)
celeryd_init.connect(tun.open)
worker_shutdown.connect(tun.close)
# ------------------------------------------------------------------------------
# DEFINE TASKS
# ------------------------------------------------------------------------------
@app.task
def directory(mypath):
return os.listdir(mypath)
@app.task
def generate(
dcd_filename,