def test_aaa_blockdetecet( self, monkey_patch, hub_blocking_detection, patching): patching.setenv('EVENTLET_NOBLOCK', '10.3') from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'eventlet']) monkey_patch.assert_called_with() hub_blocking_detection.assert_called_with(10.3, 10.3)
def test_is_patched(self): with patch('gevent.monkey.patch_all', create=True) as patch_all: import gevent gevent.version_info = (1, 0, 0) from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'gevent']) patch_all.assert_called()
def test_is_patched(self): with patch('gevent.monkey.patch_all', create=True) as patch_all: import gevent gevent.version_info = (1, 0, 0) from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'gevent']) self.assertTrue(patch_all.called)
def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection, patching): patching.setenv('EVENTLET_NOBLOCK', '10.3') from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'eventlet']) monkey_patch.assert_called_with() hub_blocking_detection.assert_called_with(10.3, 10.3)
def test_is_patched(self): with mock_module(*gevent_modules): with patch('gevent.monkey.patch_all', create=True) as patch_all: import gevent gevent.version_info = (1, 0, 0) from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'gevent']) self.assertTrue(patch_all.called)
def test_is_patched(self): self.patching.modules(*gevent_modules) patch_all = self.patching('gevent.monkey.patch_all') import gevent gevent.version_info = (1, 0, 0) from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'gevent']) patch_all.assert_called()
def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection): os.environ['EVENTLET_NOBLOCK'] = '10.3' try: from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'eventlet']) monkey_patch.assert_called_with() hub_blocking_detection.assert_called_with(10.3, 10.3) finally: os.environ.pop('EVENTLET_NOBLOCK', None)
def test_aaa_blockdetecet(self, monkey_patch, hub_blocking_detection): os.environ["EVENTLET_NOBLOCK"] = "10.3" try: from celery import maybe_patch_concurrency maybe_patch_concurrency(["x", "-P", "eventlet"]) monkey_patch.assert_called_with() hub_blocking_detection.assert_called_with(10.3, 10.3) finally: os.environ.pop("EVENTLET_NOBLOCK", None)
def main(): """Entrypoint to the ``celery`` umbrella command.""" if 'multi' not in sys.argv: maybe_patch_concurrency() from celery.bin.celery import main as _main _main()
def test_aaa_is_patched(self): with patch('eventlet.monkey_patch', create=True) as monkey_patch: from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'eventlet']) monkey_patch.assert_called_with()
def maybe_patch_concurrency(self, argv=None): argv = argv or sys.argv pool_option = self.with_pool_option(argv) if pool_option: maybe_patch_concurrency(argv, *pool_option)
def maybe_patch_concurrency(self, argv=None): argv = argv or sys.argv pool_option = self.with_pool_option(argv) if pool_option: maybe_patch_concurrency(argv, *pool_option) short_opts, long_opts = pool_option
def worker(args): """Starts Airflow Celery worker""" if not settings.validate_session(): raise SystemExit( "Worker exiting, database connection precheck failed.") autoscale = args.autoscale skip_serve_logs = args.skip_serve_logs if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") # Setup locations pid_file_path, stdout, stderr, log_file = setup_locations( process=WORKER_PROCESS_NAME, pid=args.pid, stdout=args.stdout, stderr=args.stderr, log=args.log_file, ) if hasattr(celery_app.backend, 'ResultSession'): # Pre-create the database tables now, otherwise SQLA via Celery has a # race condition where one of the subprocesses can die with "Table # already exists" error, because SQLA checks for which tables exist, # then issues a CREATE TABLE, rather than doing CREATE TABLE IF NOT # EXISTS try: session = celery_app.backend.ResultSession() session.close() except sqlalchemy.exc.IntegrityError: # At least on postgres, trying to create a table that already exist # gives a unique constraint violation or the # "pg_type_typname_nsp_index" table. If this happens we can ignore # it, we raced to create the tables and lost. pass # Setup Celery worker options = [ 'worker', '-O', 'fair', '--queues', args.queues, '--concurrency', args.concurrency, '--hostname', args.celery_hostname, '--loglevel', conf.get('logging', 'LOGGING_LEVEL'), '--pidfile', pid_file_path, ] if autoscale: options.extend(['--autoscale', autoscale]) if args.without_mingle: options.append('--without-mingle') if args.without_gossip: options.append('--without-gossip') if conf.has_option("celery", "pool"): pool = conf.get("celery", "pool") options.extend(["--pool", pool]) # Celery pools of type eventlet and gevent use greenlets, which # requires monkey patching the app: # https://eventlet.net/doc/patching.html#monkey-patch # Otherwise task instances hang on the workers and are never # executed. maybe_patch_concurrency(['-P', pool]) if args.daemon: # Run Celery worker as daemon handle = setup_logging(log_file) with open(stdout, 'w+') as stdout_handle, open(stderr, 'w+') as stderr_handle: if args.umask: umask = args.umask ctx = daemon.DaemonContext( files_preserve=[handle], umask=int(umask, 8), stdout=stdout_handle, stderr=stderr_handle, ) with ctx: sub_proc = _serve_logs(skip_serve_logs) celery_app.worker_main(options) else: # Run Celery worker in the same process sub_proc = _serve_logs(skip_serve_logs) celery_app.worker_main(options) if sub_proc: sub_proc.terminate()
def main(): setup_basic_logger() aggregator.config_from_object(app.config.scheduler) maybe_patch_concurrency() worker(aggregator).execute_from_commandline()
def worker(args): """Starts Airflow Celery worker""" if not settings.validate_session(): print("Worker exiting... database connection precheck failed! ") sys.exit(1) autoscale = args.autoscale skip_serve_logs = args.skip_serve_logs if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") # Setup locations pid_file_path, stdout, stderr, log_file = setup_locations( process=WORKER_PROCESS_NAME, pid=args.pid, stdout=args.stdout, stderr=args.stderr, log=args.log_file, ) # Setup Celery worker worker_instance = worker_bin.worker(app=celery_app) options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, 'autoscale': autoscale, 'hostname': args.celery_hostname, 'loglevel': conf.get('logging', 'LOGGING_LEVEL'), 'pidfile': pid_file_path, } if conf.has_option("celery", "pool"): pool = conf.get("celery", "pool") options["pool"] = pool # Celery pools of type eventlet and gevent use greenlets, which # requires monkey patching the app: # https://eventlet.net/doc/patching.html#monkey-patch # Otherwise task instances hang on the workers and are never # executed. maybe_patch_concurrency(['-P', pool]) if args.daemon: # Run Celery worker as daemon handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') if args.umask: umask = args.umask ctx = daemon.DaemonContext( files_preserve=[handle], umask=int(umask, 8), stdout=stdout, stderr=stderr, ) with ctx: sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) stdout.close() stderr.close() else: # Run Celery worker in the same process sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) if sub_proc: sub_proc.terminate()
def test_aaa_is_patched(self): with patch("eventlet.monkey_patch", create=True) as monkey_patch: from celery import maybe_patch_concurrency maybe_patch_concurrency(["x", "-P", "eventlet"]) monkey_patch.assert_called_with()
def main(): setup_basic_logger() aggregator.config_from_object(app.config.scheduler) maybe_patch_concurrency() beat(aggregator).execute_from_commandline()