def worker(args): """Starts Airflow Celery worker""" if not settings.validate_session(): print("Worker exiting... database connection precheck failed! ") sys.exit(1) autoscale = args.autoscale skip_serve_logs = args.skip_serve_logs if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") # Setup locations pid_file_path, stdout, stderr, log_file = setup_locations( process=WORKER_PROCESS_NAME, pid=args.pid, stdout=args.stdout, stderr=args.stderr, log=args.log_file, ) # Setup Celery worker worker_instance = worker_bin.worker(app=celery_app) options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, 'autoscale': autoscale, 'hostname': args.celery_hostname, 'loglevel': conf.get('logging', 'LOGGING_LEVEL'), 'pidfile': pid_file_path, } if conf.has_option("celery", "pool"): pool = conf.get("celery", "pool") options["pool"] = pool # Celery pools of type eventlet and gevent use greenlets, which # requires monkey patching the app: # https://eventlet.net/doc/patching.html#monkey-patch # Otherwise task instances hang on the workers and are never # executed. maybe_patch_concurrency(['-P', pool]) if args.daemon: # Run Celery worker as daemon handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') if args.umask: umask = args.umask ctx = daemon.DaemonContext( files_preserve=[handle], umask=int(umask, 8), stdout=stdout, stderr=stderr, ) with ctx: sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) stdout.close() stderr.close() else: # Run Celery worker in the same process sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) if sub_proc: sub_proc.terminate()
def worker(args): """Starts Airflow Celery worker""" if not settings.validate_session(): raise SystemExit( "Worker exiting, database connection precheck failed.") autoscale = args.autoscale skip_serve_logs = args.skip_serve_logs if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") # Setup locations pid_file_path, stdout, stderr, log_file = setup_locations( process=WORKER_PROCESS_NAME, pid=args.pid, stdout=args.stdout, stderr=args.stderr, log=args.log_file, ) if hasattr(celery_app.backend, 'ResultSession'): # Pre-create the database tables now, otherwise SQLA via Celery has a # race condition where one of the subprocesses can die with "Table # already exists" error, because SQLA checks for which tables exist, # then issues a CREATE TABLE, rather than doing CREATE TABLE IF NOT # EXISTS try: session = celery_app.backend.ResultSession() session.close() except sqlalchemy.exc.IntegrityError: # At least on postgres, trying to create a table that already exist # gives a unique constraint violation or the # "pg_type_typname_nsp_index" table. If this happens we can ignore # it, we raced to create the tables and lost. pass # Setup Celery worker worker_instance = worker_bin.worker(app=celery_app) options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, 'autoscale': autoscale, 'hostname': args.celery_hostname, 'loglevel': conf.get('logging', 'LOGGING_LEVEL'), 'pidfile': pid_file_path, } if conf.has_option("celery", "pool"): pool = conf.get("celery", "pool") options["pool"] = pool # Celery pools of type eventlet and gevent use greenlets, which # requires monkey patching the app: # https://eventlet.net/doc/patching.html#monkey-patch # Otherwise task instances hang on the workers and are never # executed. maybe_patch_concurrency(['-P', pool]) if args.daemon: # Run Celery worker as daemon handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') if args.umask: umask = args.umask ctx = daemon.DaemonContext( files_preserve=[handle], umask=int(umask, 8), stdout=stdout, stderr=stderr, ) with ctx: sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) stdout.close() stderr.close() else: # Run Celery worker in the same process sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) if sub_proc: sub_proc.terminate()
def worker(args): """Starts Airflow Celery worker""" env = os.environ.copy() env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME if not settings.validate_session(): print("Worker exiting... database connection precheck failed! ") sys.exit(1) # Celery worker from airflow.executors.celery_executor import app as celery_app from celery.bin import worker # pylint: disable=redefined-outer-name autoscale = args.autoscale if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") worker = worker.worker(app=celery_app) # pylint: disable=redefined-outer-name options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, 'autoscale': autoscale, 'hostname': args.celery_hostname, 'loglevel': conf.get('core', 'LOGGING_LEVEL'), } if conf.has_option("celery", "pool"): options["pool"] = conf.get("celery", "pool") if args.daemon: pid, stdout, stderr, log_file = setup_locations("worker", args.pid, args.stdout, args.stderr, args.log_file) handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(pid, -1), files_preserve=[handle], stdout=stdout, stderr=stderr, ) with ctx: sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True) worker.run(**options) sub_proc.kill() stdout.close() stderr.close() else: signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigint_handler) sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True) worker.run(**options) sub_proc.kill()
def webserver(args): """Starts Airflow Webserver""" print(settings.HEADER) # Check for old/insecure config, and fail safe (i.e. don't launch) if the config is wildly insecure. if conf.get('webserver', 'secret_key') == 'temporary_key': from rich import print as rich_print rich_print( "[red][bold]ERROR:[/bold] The `secret_key` setting under the webserver config has an insecure " "value - Airflow has failed safe and refuses to start. Please change this value to a new, " "per-environment, randomly generated string, for example using this command `[cyan]openssl rand " "-hex 30[/cyan]`", file=sys.stderr, ) sys.exit(1) access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile') error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile') access_logformat = args.access_logformat or conf.get('webserver', 'access_logformat') num_workers = args.workers or conf.get('webserver', 'workers') worker_timeout = args.worker_timeout or conf.get('webserver', 'web_server_worker_timeout') ssl_cert = args.ssl_cert or conf.get('webserver', 'web_server_ssl_cert') ssl_key = args.ssl_key or conf.get('webserver', 'web_server_ssl_key') if not ssl_cert and ssl_key: raise AirflowException('An SSL certificate must also be provided for use with ' + ssl_key) if ssl_cert and not ssl_key: raise AirflowException('An SSL key must also be provided for use with ' + ssl_cert) if args.debug: print(f"Starting the web server on port {args.port} and host {args.hostname}.") app = create_app(testing=conf.getboolean('core', 'unit_test_mode')) app.run( debug=True, use_reloader=not app.config['TESTING'], port=args.port, host=args.hostname, ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None, ) else: # This pre-warms the cache, and makes possible errors # get reported earlier (i.e. before demonization) os.environ['SKIP_DAGS_PARSING'] = 'True' app = cached_app(None) os.environ.pop('SKIP_DAGS_PARSING') pid_file, stdout, stderr, log_file = setup_locations( "webserver", args.pid, args.stdout, args.stderr, args.log_file ) # Check if webserver is already running if not, remove old pidfile check_if_pidfile_process_is_running(pid_file=pid_file, process_name="webserver") print( textwrap.dedent( '''\ Running the Gunicorn Server with: Workers: {num_workers} {workerclass} Host: {hostname}:{port} Timeout: {worker_timeout} Logfiles: {access_logfile} {error_logfile} Access Logformat: {access_logformat} =================================================================\ '''.format( num_workers=num_workers, workerclass=args.workerclass, hostname=args.hostname, port=args.port, worker_timeout=worker_timeout, access_logfile=access_logfile, error_logfile=error_logfile, access_logformat=access_logformat, ) ) ) run_args = [ 'gunicorn', '--workers', str(num_workers), '--worker-class', str(args.workerclass), '--timeout', str(worker_timeout), '--bind', args.hostname + ':' + str(args.port), '--name', 'airflow-webserver', '--pid', pid_file, '--config', 'python:airflow.www.gunicorn_config', ] if args.access_logfile: run_args += ['--access-logfile', str(args.access_logfile)] if args.error_logfile: run_args += ['--error-logfile', str(args.error_logfile)] if args.access_logformat and args.access_logformat.strip(): run_args += ['--access-logformat', str(args.access_logformat)] if args.daemon: run_args += ['--daemon'] if ssl_cert: run_args += ['--certfile', ssl_cert, '--keyfile', ssl_key] run_args += ["airflow.www.app:cached_app()"] gunicorn_master_proc = None def kill_proc(signum, _): # pylint: disable=unused-argument log.info("Received signal: %s. Closing gunicorn.", signum) gunicorn_master_proc.terminate() with suppress(TimeoutError): gunicorn_master_proc.wait(timeout=30) if gunicorn_master_proc.poll() is not None: gunicorn_master_proc.kill() sys.exit(0) def monitor_gunicorn(gunicorn_master_pid: int): # Register signal handlers signal.signal(signal.SIGINT, kill_proc) signal.signal(signal.SIGTERM, kill_proc) # These run forever until SIG{INT, TERM, KILL, ...} signal is sent GunicornMonitor( gunicorn_master_pid=gunicorn_master_pid, num_workers_expected=num_workers, master_timeout=conf.getint('webserver', 'web_server_master_timeout'), worker_refresh_interval=conf.getint('webserver', 'worker_refresh_interval', fallback=30), worker_refresh_batch_size=conf.getint('webserver', 'worker_refresh_batch_size', fallback=1), reload_on_plugin_change=conf.getboolean( 'webserver', 'reload_on_plugin_change', fallback=False ), ).start() if args.daemon: handle = setup_logging(log_file) base, ext = os.path.splitext(pid_file) with open(stdout, 'w+') as stdout, open(stderr, 'w+') as stderr: ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(f"{base}-monitor{ext}", -1), files_preserve=[handle], stdout=stdout, stderr=stderr, ) with ctx: subprocess.Popen(run_args, close_fds=True) # Reading pid of gunicorn master as it will be different that # the one of process spawned above. while True: sleep(0.1) gunicorn_master_proc_pid = read_pid_from_pidfile(pid_file) if gunicorn_master_proc_pid: break # Run Gunicorn monitor gunicorn_master_proc = psutil.Process(gunicorn_master_proc_pid) monitor_gunicorn(gunicorn_master_proc.pid) else: gunicorn_master_proc = subprocess.Popen(run_args, close_fds=True) monitor_gunicorn(gunicorn_master_proc.pid)
def worker(args): """Starts Airflow Celery worker""" if not settings.validate_session(): print("Worker exiting... database connection precheck failed! ") sys.exit(1) autoscale = args.autoscale skip_serve_logs = args.skip_serve_logs if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") # Setup locations pid_file_path, stdout, stderr, log_file = setup_locations( process=WORKER_PROCESS_NAME, pid=args.pid, stdout=args.stdout, stderr=args.stderr, log=args.log_file, ) # Setup Celery worker worker_instance = worker_bin.worker(app=celery_app) options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, 'autoscale': autoscale, 'hostname': args.celery_hostname, 'loglevel': conf.get('logging', 'LOGGING_LEVEL'), 'pidfile': pid_file_path, } if conf.has_option("celery", "pool"): options["pool"] = conf.get("celery", "pool") if args.daemon: # Run Celery worker as daemon handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') if args.umask: umask = args.umask ctx = daemon.DaemonContext( files_preserve=[handle], umask=int(umask, 8), stdout=stdout, stderr=stderr, ) with ctx: sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) stdout.close() stderr.close() else: # Run Celery worker in the same process sub_proc = _serve_logs(skip_serve_logs) worker_instance.run(**options) if sub_proc: sub_proc.terminate()
def webserver(args): """Starts Airflow Webserver""" print(settings.HEADER) access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile') error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile') num_workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'web_server_worker_timeout')) ssl_cert = args.ssl_cert or conf.get('webserver', 'web_server_ssl_cert') ssl_key = args.ssl_key or conf.get('webserver', 'web_server_ssl_key') if not ssl_cert and ssl_key: raise AirflowException( 'An SSL certificate must also be provided for use with ' + ssl_key) if ssl_cert and not ssl_key: raise AirflowException( 'An SSL key must also be provided for use with ' + ssl_cert) if args.debug: print("Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app = create_app(testing=conf.getboolean('core', 'unit_test_mode')) app.run(debug=True, use_reloader=not app.config['TESTING'], port=args.port, host=args.hostname, ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None) else: # This pre-warms the cache, and makes possible errors # get reported earlier (i.e. before demonization) os.environ['SKIP_DAGS_PARSING'] = 'True' app = cached_app(None) os.environ.pop('SKIP_DAGS_PARSING') pid_file, stdout, stderr, log_file = setup_locations( "webserver", args.pid, args.stdout, args.stderr, args.log_file) # Check if webserver is already running if not, remove old pidfile check_if_pidfile_process_is_running(pid_file=pid_file, process_name="webserver") print( textwrap.dedent('''\ Running the Gunicorn Server with: Workers: {num_workers} {workerclass} Host: {hostname}:{port} Timeout: {worker_timeout} Logfiles: {access_logfile} {error_logfile} =================================================================\ '''.format(num_workers=num_workers, workerclass=args.workerclass, hostname=args.hostname, port=args.port, worker_timeout=worker_timeout, access_logfile=access_logfile, error_logfile=error_logfile))) run_args = [ 'gunicorn', '--workers', str(num_workers), '--worker-class', str(args.workerclass), '--timeout', str(worker_timeout), '--bind', args.hostname + ':' + str(args.port), '--name', 'airflow-webserver', '--pid', pid_file, '--config', 'python:airflow.www.gunicorn_config', ] if args.access_logfile: run_args += ['--access-logfile', str(args.access_logfile)] if args.error_logfile: run_args += ['--error-logfile', str(args.error_logfile)] if args.daemon: run_args += ['--daemon'] if ssl_cert: run_args += ['--certfile', ssl_cert, '--keyfile', ssl_key] run_args += ["airflow.www.app:cached_app()"] gunicorn_master_proc = None def kill_proc(dummy_signum, dummy_frame): # pylint: disable=unused-argument gunicorn_master_proc.terminate() gunicorn_master_proc.wait() sys.exit(0) def monitor_gunicorn(gunicorn_master_proc): # Register signal handlers signal.signal(signal.SIGINT, kill_proc) signal.signal(signal.SIGTERM, kill_proc) # These run forever until SIG{INT, TERM, KILL, ...} signal is sent GunicornMonitor( gunicorn_master_proc=gunicorn_master_proc, num_workers_expected=num_workers, master_timeout=conf.getint('webserver', 'web_server_master_timeout'), worker_refresh_interval=conf.getint('webserver', 'worker_refresh_interval', fallback=10), worker_refresh_batch_size=conf.getint( 'webserver', 'worker_refresh_batch_size', fallback=1), reload_on_plugin_change=conf.getboolean( 'webserver', 'reload_on_plugin_change', fallback=False), ).start() if args.daemon: handle = setup_logging(log_file) base, ext = os.path.splitext(pid_file) with open(stdout, 'w+') as stdout, open(stderr, 'w+') as stderr: ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(f"{base}-monitor{ext}", -1), files_preserve=[handle], stdout=stdout, stderr=stderr, ) with ctx: subprocess.Popen(run_args, close_fds=True) # Reading pid of gunicorn master as it will be different that # the one of process spawned above. while True: sleep(0.1) gunicorn_master_proc_pid = read_pid_from_pidfile( pid_file) if gunicorn_master_proc_pid: break # Run Gunicorn monitor gunicorn_master_proc = psutil.Process( gunicorn_master_proc_pid) monitor_gunicorn(gunicorn_master_proc) else: gunicorn_master_proc = subprocess.Popen(run_args, close_fds=True) monitor_gunicorn(gunicorn_master_proc)
def test_setup_locations_none_pid_path(self): process_name = "fake_process" default_pid_path = os.path.join(settings.AIRFLOW_HOME, f"airflow-{process_name}.pid") pid, _, _, _ = cli.setup_locations(process=process_name) assert pid == default_pid_path
def test_setup_locations_absolute_pid_path(self): abs_pid_path = os.path.join(os.getcwd(), "fake.pid") pid, _, _, _ = cli.setup_locations(process="fake_process", pid=abs_pid_path) assert pid == abs_pid_path
def test_setup_locations_relative_pid_path(self): relative_pid_path = "fake.pid" pid_full_path = os.path.join(os.getcwd(), relative_pid_path) pid, _, _, _ = cli.setup_locations(process="fake_process", pid=relative_pid_path) assert pid == pid_full_path
def webserver(args): """Starts Airflow Webserver""" print(settings.HEADER) access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile') error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile') num_workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'web_server_worker_timeout')) ssl_cert = args.ssl_cert or conf.get('webserver', 'web_server_ssl_cert') ssl_key = args.ssl_key or conf.get('webserver', 'web_server_ssl_key') if not ssl_cert and ssl_key: raise AirflowException( 'An SSL certificate must also be provided for use with ' + ssl_key) if ssl_cert and not ssl_key: raise AirflowException( 'An SSL key must also be provided for use with ' + ssl_cert) if args.debug: print("Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app, _ = create_app(None, testing=conf.getboolean('core', 'unit_test_mode')) app.run(debug=True, use_reloader=not app.config['TESTING'], port=args.port, host=args.hostname, ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None) else: os.environ['SKIP_DAGS_PARSING'] = 'True' app = cached_app(None) pid, stdout, stderr, log_file = setup_locations( "webserver", args.pid, args.stdout, args.stderr, args.log_file) os.environ.pop('SKIP_DAGS_PARSING') if args.daemon: handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') print( textwrap.dedent('''\ Running the Gunicorn Server with: Workers: {num_workers} {workerclass} Host: {hostname}:{port} Timeout: {worker_timeout} Logfiles: {access_logfile} {error_logfile} =================================================================\ '''.format(num_workers=num_workers, workerclass=args.workerclass, hostname=args.hostname, port=args.port, worker_timeout=worker_timeout, access_logfile=access_logfile, error_logfile=error_logfile))) run_args = [ 'gunicorn', '-w', str(num_workers), '-k', str(args.workerclass), '-t', str(worker_timeout), '-b', args.hostname + ':' + str(args.port), '-n', 'airflow-webserver', '-p', str(pid), '-c', 'python:airflow.www.gunicorn_config', ] if args.access_logfile: run_args += ['--access-logfile', str(args.access_logfile)] if args.error_logfile: run_args += ['--error-logfile', str(args.error_logfile)] if args.daemon: run_args += ['-D'] if ssl_cert: run_args += ['--certfile', ssl_cert, '--keyfile', ssl_key] webserver_module = 'www' run_args += ["airflow." + webserver_module + ".app:cached_app()"] gunicorn_master_proc = None def kill_proc(dummy_signum, dummy_frame): # pylint: disable=unused-argument gunicorn_master_proc.terminate() gunicorn_master_proc.wait() sys.exit(0) def monitor_gunicorn(gunicorn_master_proc): # These run forever until SIG{INT, TERM, KILL, ...} signal is sent if conf.getint('webserver', 'worker_refresh_interval') > 0: master_timeout = conf.getint('webserver', 'web_server_master_timeout') restart_workers(gunicorn_master_proc, num_workers, master_timeout) else: while gunicorn_master_proc.poll() is None: time.sleep(1) sys.exit(gunicorn_master_proc.returncode) if args.daemon: base, ext = os.path.splitext(pid) ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(base + "-monitor" + ext, -1), files_preserve=[handle], stdout=stdout, stderr=stderr, signal_map={ signal.SIGINT: kill_proc, signal.SIGTERM: kill_proc }, ) with ctx: subprocess.Popen(run_args, close_fds=True) # Reading pid file directly, since Popen#pid doesn't # seem to return the right value with DaemonContext. while True: try: with open(pid) as file: gunicorn_master_proc_pid = int(file.read()) break except OSError: log.debug( "Waiting for gunicorn's pid file to be created.") time.sleep(0.1) gunicorn_master_proc = psutil.Process(gunicorn_master_proc_pid) monitor_gunicorn(gunicorn_master_proc) stdout.close() stderr.close() else: gunicorn_master_proc = subprocess.Popen(run_args, close_fds=True) signal.signal(signal.SIGINT, kill_proc) signal.signal(signal.SIGTERM, kill_proc) monitor_gunicorn(gunicorn_master_proc)
def worker(args): """Starts Airflow Celery worker""" # Disable connection pool so that celery worker does not hold an unnecessary db connection settings.reconfigure_orm(disable_connection_pool=True) if not settings.validate_session(): raise SystemExit( "Worker exiting, database connection precheck failed.") autoscale = args.autoscale skip_serve_logs = args.skip_serve_logs if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") # Setup locations pid_file_path, stdout, stderr, log_file = setup_locations( process=WORKER_PROCESS_NAME, pid=args.pid, stdout=args.stdout, stderr=args.stderr, log=args.log_file, ) if hasattr(celery_app.backend, 'ResultSession'): # Pre-create the database tables now, otherwise SQLA via Celery has a # race condition where one of the subprocesses can die with "Table # already exists" error, because SQLA checks for which tables exist, # then issues a CREATE TABLE, rather than doing CREATE TABLE IF NOT # EXISTS try: session = celery_app.backend.ResultSession() session.close() except sqlalchemy.exc.IntegrityError: # At least on postgres, trying to create a table that already exist # gives a unique constraint violation or the # "pg_type_typname_nsp_index" table. If this happens we can ignore # it, we raced to create the tables and lost. pass # backwards-compatible: https://github.com/apache/airflow/pull/21506#pullrequestreview-879893763 celery_log_level = conf.get('logging', 'CELERY_LOGGING_LEVEL') if not celery_log_level: celery_log_level = conf.get('logging', 'LOGGING_LEVEL') # Setup Celery worker options = [ 'worker', '-O', 'fair', '--queues', args.queues, '--concurrency', args.concurrency, '--hostname', args.celery_hostname, '--loglevel', celery_log_level, '--pidfile', pid_file_path, ] if autoscale: options.extend(['--autoscale', autoscale]) if args.without_mingle: options.append('--without-mingle') if args.without_gossip: options.append('--without-gossip') if conf.has_option("celery", "pool"): pool = conf.get("celery", "pool") options.extend(["--pool", pool]) # Celery pools of type eventlet and gevent use greenlets, which # requires monkey patching the app: # https://eventlet.net/doc/patching.html#monkey-patch # Otherwise task instances hang on the workers and are never # executed. maybe_patch_concurrency(['-P', pool]) if args.daemon: # Run Celery worker as daemon handle = setup_logging(log_file) with open(stdout, 'w+') as stdout_handle, open(stderr, 'w+') as stderr_handle: if args.umask: umask = args.umask ctx = daemon.DaemonContext( files_preserve=[handle], umask=int(umask, 8), stdout=stdout_handle, stderr=stderr_handle, ) with ctx: _run_worker(options=options, skip_serve_logs=skip_serve_logs) else: # Run Celery worker in the same process _run_worker(options=options, skip_serve_logs=skip_serve_logs)