def test_aaa_blockdetecet(
self, monkey_patch, hub_blocking_detection, patching):
patching.setenv('EVENTLET_NOBLOCK', '10.3')
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'eventlet'])
monkey_patch.assert_called_with()
hub_blocking_detection.assert_called_with(10.3, 10.3)
def test_is_patched(self):
with patch('gevent.monkey.patch_all', create=True) as patch_all:
import gevent
gevent.version_info = (1, 0, 0)
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'gevent'])
patch_all.assert_called()
def test_is_patched(self):
with patch('gevent.monkey.patch_all', create=True) as patch_all:
import gevent
gevent.version_info = (1, 0, 0)
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'gevent'])
self.assertTrue(patch_all.called)
def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection,
patching):
patching.setenv('EVENTLET_NOBLOCK', '10.3')
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'eventlet'])
monkey_patch.assert_called_with()
hub_blocking_detection.assert_called_with(10.3, 10.3)
def test_is_patched(self):
with mock_module(*gevent_modules):
with patch('gevent.monkey.patch_all', create=True) as patch_all:
import gevent
gevent.version_info = (1, 0, 0)
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'gevent'])
self.assertTrue(patch_all.called)
def test_is_patched(self):
self.patching.modules(*gevent_modules)
patch_all = self.patching('gevent.monkey.patch_all')
import gevent
gevent.version_info = (1, 0, 0)
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'gevent'])
patch_all.assert_called()
def test_is_patched(self):
self.patching.modules(*gevent_modules)
patch_all = self.patching('gevent.monkey.patch_all')
import gevent
gevent.version_info = (1, 0, 0)
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'gevent'])
patch_all.assert_called()
def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection):
os.environ['EVENTLET_NOBLOCK'] = '10.3'
try:
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'eventlet'])
monkey_patch.assert_called_with()
hub_blocking_detection.assert_called_with(10.3, 10.3)
finally:
os.environ.pop('EVENTLET_NOBLOCK', None)
def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection):
os.environ['EVENTLET_NOBLOCK'] = '10.3'
try:
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'eventlet'])
monkey_patch.assert_called_with()
hub_blocking_detection.assert_called_with(10.3, 10.3)
finally:
os.environ.pop('EVENTLET_NOBLOCK', None)
def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection):
os.environ["EVENTLET_NOBLOCK"] = "10.3"
try:
from celery import maybe_patch_concurrency
maybe_patch_concurrency(["x", "-P", "eventlet"])
monkey_patch.assert_called_with()
hub_blocking_detection.assert_called_with(10.3, 10.3)
finally:
os.environ.pop("EVENTLET_NOBLOCK", None)
def main():
"""Entrypoint to the ``celery`` umbrella command."""
if 'multi' not in sys.argv:
maybe_patch_concurrency()
from celery.bin.celery import main as _main
_main()
def test_aaa_is_patched(self):
with patch('eventlet.monkey_patch', create=True) as monkey_patch:
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'eventlet'])
monkey_patch.assert_called_with()
def maybe_patch_concurrency(self, argv=None):
argv = argv or sys.argv
pool_option = self.with_pool_option(argv)
if pool_option:
maybe_patch_concurrency(argv, *pool_option)
def maybe_patch_concurrency(self, argv=None):
argv = argv or sys.argv
pool_option = self.with_pool_option(argv)
if pool_option:
maybe_patch_concurrency(argv, *pool_option)
short_opts, long_opts = pool_option
def worker(args):
"""Starts Airflow Celery worker"""
if not settings.validate_session():
raise SystemExit(
"Worker exiting, database connection precheck failed.")
autoscale = args.autoscale
skip_serve_logs = args.skip_serve_logs
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
# Setup locations
pid_file_path, stdout, stderr, log_file = setup_locations(
process=WORKER_PROCESS_NAME,
pid=args.pid,
stdout=args.stdout,
stderr=args.stderr,
log=args.log_file,
)
if hasattr(celery_app.backend, 'ResultSession'):
# Pre-create the database tables now, otherwise SQLA via Celery has a
# race condition where one of the subprocesses can die with "Table
# already exists" error, because SQLA checks for which tables exist,
# then issues a CREATE TABLE, rather than doing CREATE TABLE IF NOT
# EXISTS
try:
session = celery_app.backend.ResultSession()
session.close()
except sqlalchemy.exc.IntegrityError:
# At least on postgres, trying to create a table that already exist
# gives a unique constraint violation or the
# "pg_type_typname_nsp_index" table. If this happens we can ignore
# it, we raced to create the tables and lost.
pass
# Setup Celery worker
options = [
'worker',
'-O',
'fair',
'--queues',
args.queues,
'--concurrency',
args.concurrency,
'--hostname',
args.celery_hostname,
'--loglevel',
conf.get('logging', 'LOGGING_LEVEL'),
'--pidfile',
pid_file_path,
]
if autoscale:
options.extend(['--autoscale', autoscale])
if args.without_mingle:
options.append('--without-mingle')
if args.without_gossip:
options.append('--without-gossip')
if conf.has_option("celery", "pool"):
pool = conf.get("celery", "pool")
options.extend(["--pool", pool])
# Celery pools of type eventlet and gevent use greenlets, which
# requires monkey patching the app:
# https://eventlet.net/doc/patching.html#monkey-patch
# Otherwise task instances hang on the workers and are never
# executed.
maybe_patch_concurrency(['-P', pool])
if args.daemon:
# Run Celery worker as daemon
handle = setup_logging(log_file)
with open(stdout, 'w+') as stdout_handle, open(stderr,
'w+') as stderr_handle:
if args.umask:
umask = args.umask
ctx = daemon.DaemonContext(
files_preserve=[handle],
umask=int(umask, 8),
stdout=stdout_handle,
stderr=stderr_handle,
)
with ctx:
sub_proc = _serve_logs(skip_serve_logs)
celery_app.worker_main(options)
else:
# Run Celery worker in the same process
sub_proc = _serve_logs(skip_serve_logs)
celery_app.worker_main(options)
if sub_proc:
sub_proc.terminate()
def test_aaa_is_patched(self):
with patch('eventlet.monkey_patch', create=True) as monkey_patch:
from celery import maybe_patch_concurrency
maybe_patch_concurrency(['x', '-P', 'eventlet'])
monkey_patch.assert_called_with()
def main():
setup_basic_logger()
aggregator.config_from_object(app.config.scheduler)
maybe_patch_concurrency()
worker(aggregator).execute_from_commandline()
def worker(args):
"""Starts Airflow Celery worker"""
if not settings.validate_session():
print("Worker exiting... database connection precheck failed! ")
sys.exit(1)
autoscale = args.autoscale
skip_serve_logs = args.skip_serve_logs
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
# Setup locations
pid_file_path, stdout, stderr, log_file = setup_locations(
process=WORKER_PROCESS_NAME,
pid=args.pid,
stdout=args.stdout,
stderr=args.stderr,
log=args.log_file,
)
# Setup Celery worker
worker_instance = worker_bin.worker(app=celery_app)
options = {
'optimization': 'fair',
'O': 'fair',
'queues': args.queues,
'concurrency': args.concurrency,
'autoscale': autoscale,
'hostname': args.celery_hostname,
'loglevel': conf.get('logging', 'LOGGING_LEVEL'),
'pidfile': pid_file_path,
}
if conf.has_option("celery", "pool"):
pool = conf.get("celery", "pool")
options["pool"] = pool
# Celery pools of type eventlet and gevent use greenlets, which
# requires monkey patching the app:
# https://eventlet.net/doc/patching.html#monkey-patch
# Otherwise task instances hang on the workers and are never
# executed.
maybe_patch_concurrency(['-P', pool])
if args.daemon:
# Run Celery worker as daemon
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
if args.umask:
umask = args.umask
ctx = daemon.DaemonContext(
files_preserve=[handle],
umask=int(umask, 8),
stdout=stdout,
stderr=stderr,
)
with ctx:
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
stdout.close()
stderr.close()
else:
# Run Celery worker in the same process
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
if sub_proc:
sub_proc.terminate()
def test_aaa_is_patched(self):
with patch("eventlet.monkey_patch", create=True) as monkey_patch:
from celery import maybe_patch_concurrency
maybe_patch_concurrency(["x", "-P", "eventlet"])
monkey_patch.assert_called_with()
def main():
setup_basic_logger()
aggregator.config_from_object(app.config.scheduler)
maybe_patch_concurrency()
beat(aggregator).execute_from_commandline()
