def scheduler(args):
"""Starts Airflow Scheduler"""
skip_serve_logs = args.skip_serve_logs
print(settings.HEADER)
job = SchedulerJob(
subdir=process_subdir(args.subdir),
num_runs=args.num_runs,
do_pickle=args.do_pickle,
)
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"scheduler", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
with open(stdout, 'w+') as stdout_handle, open(stderr,
'w+') as stderr_handle:
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout_handle,
stderr=stderr_handle,
)
with ctx:
sub_proc = _serve_logs(skip_serve_logs)
job.run()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
sub_proc = _serve_logs(skip_serve_logs)
job.run()
if sub_proc:
sub_proc.terminate()
def scheduler(args):
"""Starts Airflow Scheduler"""
print(settings.HEADER)
print(get_config_with_source())
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"scheduler", args.pid, args.stdout, args.stderr, args.log_file
)
handle = setup_logging(log_file)
with open(stdout, 'w+') as stdout_handle, open(stderr, 'w+') as stderr_handle:
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout_handle,
stderr=stderr_handle,
)
with ctx:
_run_scheduler_job(args=args)
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
signal.signal(signal.SIGUSR1, sigconf_handler)
_run_scheduler_job(args=args)
def triggerer(args):
"""Starts Airflow Triggerer"""
settings.MASK_SECRETS_IN_LOGS = True
print(settings.HEADER)
job = TriggererJob(capacity=args.capacity)
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"triggerer", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
with open(stdout, 'w+') as stdout_handle, open(stderr,
'w+') as stderr_handle:
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout_handle,
stderr=stderr_handle,
)
with ctx:
job.run()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
job.run()
def event_based_scheduler(args):
"""Starts Airflow Event-based Scheduler"""
print(settings.HEADER)
job = EventBasedSchedulerJob(
dag_directory=process_subdir(args.subdir),
server_uri=args.server_uri,
)
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"event_based_scheduler", args.pid, args.stdout, args.stderr,
args.log_file)
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
job.run()
stdout.close()
stderr.close()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
job.run()
def dag_processor(args):
"""Starts Airflow Dag Processor Job"""
if not conf.getboolean("scheduler", "standalone_dag_processor"):
raise SystemExit(
'The option [scheduler/standalone_dag_processor] must be True.')
sql_conn: str = conf.get('database', 'sql_alchemy_conn').lower()
if sql_conn.startswith('sqlite'):
raise SystemExit(
'Standalone DagProcessor is not supported when using sqlite.')
manager = _create_dag_processor_manager(args)
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"dag-processor", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
with open(stdout, 'w+') as stdout_handle, open(stderr,
'w+') as stderr_handle:
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout_handle,
stderr=stderr_handle,
)
with ctx:
try:
manager.register_exit_signals()
manager.start()
finally:
manager.terminate()
manager.end()
else:
manager.register_exit_signals()
manager.start()
def scheduler(args):
"""Starts Airflow Scheduler"""
print(settings.HEADER)
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"scheduler", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
with open(stdout, 'a') as stdout_handle, open(stderr,
'a') as stderr_handle:
stdout_handle.truncate(0)
stderr_handle.truncate(0)
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout_handle,
stderr=stderr_handle,
umask=int(settings.DAEMON_UMASK, 8),
)
with ctx:
_run_scheduler_job(args=args)
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
_run_scheduler_job(args=args)
def scheduler(args):
"""Starts Airflow Scheduler"""
print(settings.HEADER)
job = SchedulerJob(subdir=process_subdir(args.subdir),
num_runs=args.num_runs,
do_pickle=args.do_pickle)
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"scheduler", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
job.run()
stdout.close()
stderr.close()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
job.run()
def worker(args):
"""Starts Airflow Celery worker"""
env = os.environ.copy()
env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME
if not settings.validate_session():
print("Worker exiting... database connection precheck failed! ")
sys.exit(1)
autoscale = args.autoscale
skip_serve_logs = args.skip_serve_logs
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
worker_instance = worker_bin.worker(app=celery_app)
options = {
'optimization': 'fair',
'O': 'fair',
'queues': args.queues,
'concurrency': args.concurrency,
'autoscale': autoscale,
'hostname': args.celery_hostname,
'loglevel': conf.get('logging', 'LOGGING_LEVEL'),
}
if conf.has_option("celery", "pool"):
options["pool"] = conf.get("celery", "pool")
if args.daemon:
pid, stdout, stderr, log_file = setup_locations("worker",
args.pid,
args.stdout,
args.stderr,
args.log_file)
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
stdout.close()
stderr.close()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
if sub_proc:
sub_proc.terminate()
def scheduler(args):
"""Starts Airflow Scheduler"""
print(settings.HEADER)
job = SchedulerJob(
subdir=process_subdir(args.subdir),
num_runs=args.num_runs,
do_pickle=args.do_pickle,
)
scheduler_name = SchedulerFactory.get_scheduler_name()
if scheduler_name == SchedulerFactory.DEFAULT_SCHEDULER:
pass
elif scheduler_name == SchedulerFactory.EVENT_BASED_SCHEDULER:
job = EventBasedSchedulerJob(dag_directory=process_subdir(args.subdir),
server_uri=args.server_uri)
else:
scheduler_class = SchedulerFactory.get_default_scheduler()
job = scheduler_class()
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"scheduler", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
job.run()
stdout.close()
stderr.close()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
job.run()
def triggerer(args):
"""Starts Airflow Triggerer"""
settings.MASK_SECRETS_IN_LOGS = True
print(settings.HEADER)
job = TriggererJob(capacity=args.capacity)
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"triggerer", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
with open(stdout, 'w+') as stdout_handle, open(stderr,
'w+') as stderr_handle:
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout_handle,
stderr=stderr_handle,
)
with ctx:
job.run()
else:
# There is a bug in CPython (fixed in March 2022 but not yet released) that
# makes async.io handle SIGTERM improperly by using async unsafe
# functions and hanging the triggerer receive SIGPIPE while handling
# SIGTERN/SIGINT and deadlocking itself. Until the bug is handled
# we should rather rely on standard handling of the signals rather than
# adding our own signal handlers. Seems that even if our signal handler
# just run exit(0) - it caused a race condition that led to the hanging.
#
# More details:
# * https://bugs.python.org/issue39622
# * https://github.com/python/cpython/issues/83803
#
# signal.signal(signal.SIGINT, sigint_handler)
# signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
job.run()
def webserver(args):
"""Starts Airflow Webserver"""
print(settings.HEADER)
access_logfile = args.access_logfile or conf.get('webserver',
'access_logfile')
error_logfile = args.error_logfile or conf.get('webserver',
'error_logfile')
num_workers = args.workers or conf.get('webserver', 'workers')
worker_timeout = args.worker_timeout or conf.get(
'webserver', 'web_server_worker_timeout')
ssl_cert = args.ssl_cert or conf.get('webserver', 'web_server_ssl_cert')
ssl_key = args.ssl_key or conf.get('webserver', 'web_server_ssl_key')
if not ssl_cert and ssl_key:
raise AirflowException(
'An SSL certificate must also be provided for use with ' + ssl_key)
if ssl_cert and not ssl_key:
raise AirflowException(
'An SSL key must also be provided for use with ' + ssl_cert)
if args.debug:
print(
f"Starting the web server on port {args.port} and host {args.hostname}."
)
app = create_app(testing=conf.getboolean('core', 'unit_test_mode'))
app.run(
debug=True,
use_reloader=not app.config['TESTING'],
port=args.port,
host=args.hostname,
ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None,
)
else:
# This pre-warms the cache, and makes possible errors
# get reported earlier (i.e. before demonization)
os.environ['SKIP_DAGS_PARSING'] = 'True'
app = cached_app(None)
os.environ.pop('SKIP_DAGS_PARSING')
pid_file, stdout, stderr, log_file = setup_locations(
"webserver", args.pid, args.stdout, args.stderr, args.log_file)
# Check if webserver is already running if not, remove old pidfile
check_if_pidfile_process_is_running(pid_file=pid_file,
process_name="webserver")
print(
textwrap.dedent('''\
Running the Gunicorn Server with:
Workers: {num_workers} {workerclass}
Host: {hostname}:{port}
Timeout: {worker_timeout}
Logfiles: {access_logfile} {error_logfile}
=================================================================\
'''.format(
num_workers=num_workers,
workerclass=args.workerclass,
hostname=args.hostname,
port=args.port,
worker_timeout=worker_timeout,
access_logfile=access_logfile,
error_logfile=error_logfile,
)))
run_args = [
'gunicorn',
'--workers',
str(num_workers),
'--worker-class',
str(args.workerclass),
'--timeout',
str(worker_timeout),
'--bind',
args.hostname + ':' + str(args.port),
'--name',
'airflow-webserver',
'--pid',
pid_file,
'--config',
'python:airflow.www.gunicorn_config',
]
if args.access_logfile:
run_args += ['--access-logfile', str(args.access_logfile)]
if args.error_logfile:
run_args += ['--error-logfile', str(args.error_logfile)]
if args.daemon:
run_args += ['--daemon']
if ssl_cert:
run_args += ['--certfile', ssl_cert, '--keyfile', ssl_key]
run_args += ["airflow.www.app:cached_app()"]
gunicorn_master_proc = None
def kill_proc(signum, _): # pylint: disable=unused-argument
log.info("Received signal: %s. Closing gunicorn.", signum)
gunicorn_master_proc.terminate()
with suppress(TimeoutError):
gunicorn_master_proc.wait(timeout=30)
if gunicorn_master_proc.poll() is not None:
gunicorn_master_proc.kill()
sys.exit(0)
def monitor_gunicorn(gunicorn_master_pid: int):
# Register signal handlers
signal.signal(signal.SIGINT, kill_proc)
signal.signal(signal.SIGTERM, kill_proc)
# These run forever until SIG{INT, TERM, KILL, ...} signal is sent
GunicornMonitor(
gunicorn_master_pid=gunicorn_master_pid,
num_workers_expected=num_workers,
master_timeout=conf.getint('webserver',
'web_server_master_timeout'),
worker_refresh_interval=conf.getint('webserver',
'worker_refresh_interval',
fallback=30),
worker_refresh_batch_size=conf.getint(
'webserver', 'worker_refresh_batch_size', fallback=1),
reload_on_plugin_change=conf.getboolean(
'webserver', 'reload_on_plugin_change', fallback=False),
).start()
if args.daemon:
handle = setup_logging(log_file)
base, ext = os.path.splitext(pid_file)
with open(stdout, 'w+') as stdout, open(stderr, 'w+') as stderr:
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(f"{base}-monitor{ext}", -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
subprocess.Popen(run_args, close_fds=True)
# Reading pid of gunicorn master as it will be different that
# the one of process spawned above.
while True:
sleep(0.1)
gunicorn_master_proc_pid = read_pid_from_pidfile(
pid_file)
if gunicorn_master_proc_pid:
break
# Run Gunicorn monitor
gunicorn_master_proc = psutil.Process(
gunicorn_master_proc_pid)
monitor_gunicorn(gunicorn_master_proc.pid)
else:
gunicorn_master_proc = subprocess.Popen(run_args, close_fds=True)
monitor_gunicorn(gunicorn_master_proc.pid)
def webserver(args):
"""Starts Airflow Webserver"""
print(settings.HEADER)
access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile')
error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile')
num_workers = args.workers or conf.get('webserver', 'workers')
worker_timeout = (args.worker_timeout or
conf.get('webserver', 'web_server_worker_timeout'))
ssl_cert = args.ssl_cert or conf.get('webserver', 'web_server_ssl_cert')
ssl_key = args.ssl_key or conf.get('webserver', 'web_server_ssl_key')
if not ssl_cert and ssl_key:
raise AirflowException(
'An SSL certificate must also be provided for use with ' + ssl_key)
if ssl_cert and not ssl_key:
raise AirflowException(
'An SSL key must also be provided for use with ' + ssl_cert)
if args.debug:
print(
"Starting the web server on port {0} and host {1}.".format(
args.port, args.hostname))
app, _ = create_app(None, testing=conf.getboolean('core', 'unit_test_mode'))
app.run(debug=True, use_reloader=not app.config['TESTING'],
port=args.port, host=args.hostname,
ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None)
else:
os.environ['SKIP_DAGS_PARSING'] = 'True'
app = cached_app(None)
pid, stdout, stderr, log_file = setup_locations(
"webserver", args.pid, args.stdout, args.stderr, args.log_file)
os.environ.pop('SKIP_DAGS_PARSING')
if args.daemon:
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
print(
textwrap.dedent('''\
Running the Gunicorn Server with:
Workers: {num_workers} {workerclass}
Host: {hostname}:{port}
Timeout: {worker_timeout}
Logfiles: {access_logfile} {error_logfile}
=================================================================\
'''.format(num_workers=num_workers, workerclass=args.workerclass,
hostname=args.hostname, port=args.port,
worker_timeout=worker_timeout, access_logfile=access_logfile,
error_logfile=error_logfile)))
run_args = [
'gunicorn',
'-w', str(num_workers),
'-k', str(args.workerclass),
'-t', str(worker_timeout),
'-b', args.hostname + ':' + str(args.port),
'-n', 'airflow-webserver',
'-p', str(pid),
'-c', 'python:airflow.www.gunicorn_config',
]
if args.access_logfile:
run_args += ['--access-logfile', str(args.access_logfile)]
if args.error_logfile:
run_args += ['--error-logfile', str(args.error_logfile)]
if args.daemon:
run_args += ['-D']
if ssl_cert:
run_args += ['--certfile', ssl_cert, '--keyfile', ssl_key]
webserver_module = 'www'
run_args += ["airflow." + webserver_module + ".app:cached_app()"]
gunicorn_master_proc = None
def kill_proc(dummy_signum, dummy_frame): # pylint: disable=unused-argument
gunicorn_master_proc.terminate()
gunicorn_master_proc.wait()
sys.exit(0)
def monitor_gunicorn(gunicorn_master_proc):
# These run forever until SIG{INT, TERM, KILL, ...} signal is sent
if conf.getint('webserver', 'worker_refresh_interval') > 0:
master_timeout = conf.getint('webserver', 'web_server_master_timeout')
restart_workers(gunicorn_master_proc, num_workers, master_timeout)
else:
while gunicorn_master_proc.poll() is None:
time.sleep(1)
sys.exit(gunicorn_master_proc.returncode)
if args.daemon:
base, ext = os.path.splitext(pid)
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(base + "-monitor" + ext, -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
signal_map={
signal.SIGINT: kill_proc,
signal.SIGTERM: kill_proc
},
)
with ctx:
subprocess.Popen(run_args, close_fds=True)
# Reading pid file directly, since Popen#pid doesn't
# seem to return the right value with DaemonContext.
while True:
try:
with open(pid) as file:
gunicorn_master_proc_pid = int(file.read())
break
except OSError:
log.debug("Waiting for gunicorn's pid file to be created.")
time.sleep(0.1)
gunicorn_master_proc = psutil.Process(gunicorn_master_proc_pid)
monitor_gunicorn(gunicorn_master_proc)
stdout.close()
stderr.close()
else:
gunicorn_master_proc = subprocess.Popen(run_args, close_fds=True)
signal.signal(signal.SIGINT, kill_proc)
signal.signal(signal.SIGTERM, kill_proc)
monitor_gunicorn(gunicorn_master_proc)
def webserver(args):
"""Starts Airflow Webserver"""
print(settings.HEADER)
# Check for old/insecure config, and fail safe (i.e. don't launch) if the config is wildly insecure.
if conf.get('webserver', 'secret_key') == 'temporary_key':
from rich import print as rich_print
rich_print(
"[red][bold]ERROR:[/bold] The `secret_key` setting under the webserver config has an insecure "
"value - Airflow has failed safe and refuses to start. Please change this value to a new, "
"per-environment, randomly generated string, for example using this command `[cyan]openssl rand "
"-hex 30[/cyan]`",
file=sys.stderr,
)
sys.exit(1)
access_logfile = args.access_logfile or conf.get('webserver',
'access_logfile')
error_logfile = args.error_logfile or conf.get('webserver',
'error_logfile')
access_logformat = args.access_logformat or conf.get(
'webserver', 'access_logformat')
num_workers = args.workers or conf.get('webserver', 'workers')
worker_timeout = args.worker_timeout or conf.get(
'webserver', 'web_server_worker_timeout')
ssl_cert = args.ssl_cert or conf.get('webserver', 'web_server_ssl_cert')
ssl_key = args.ssl_key or conf.get('webserver', 'web_server_ssl_key')
if not ssl_cert and ssl_key:
raise AirflowException(
'An SSL certificate must also be provided for use with ' + ssl_key)
if ssl_cert and not ssl_key:
raise AirflowException(
'An SSL key must also be provided for use with ' + ssl_cert)
if args.debug:
print(
f"Starting the web server on port {args.port} and host {args.hostname}."
)
app = create_app(testing=conf.getboolean('core', 'unit_test_mode'))
app.run(
debug=True,
use_reloader=not app.config['TESTING'],
port=args.port,
host=args.hostname,
ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None,
)
else:
pid_file, stdout, stderr, log_file = setup_locations(
"webserver", args.pid, args.stdout, args.stderr, args.log_file)
# Check if webserver is already running if not, remove old pidfile
check_if_pidfile_process_is_running(pid_file=pid_file,
process_name="webserver")
print(
textwrap.dedent(f'''\
Running the Gunicorn Server with:
Workers: {num_workers} {args.workerclass}
Host: {args.hostname}:{args.port}
Timeout: {worker_timeout}
Logfiles: {access_logfile} {error_logfile}
Access Logformat: {access_logformat}
================================================================='''
))
run_args = [
sys.executable,
'-m',
'gunicorn',
'--workers',
str(num_workers),
'--worker-class',
str(args.workerclass),
'--timeout',
str(worker_timeout),
'--bind',
args.hostname + ':' + str(args.port),
'--name',
'airflow-webserver',
'--pid',
pid_file,
'--config',
'python:airflow.www.gunicorn_config',
]
if args.access_logfile:
run_args += ['--access-logfile', str(args.access_logfile)]
if args.error_logfile:
run_args += ['--error-logfile', str(args.error_logfile)]
if args.access_logformat and args.access_logformat.strip():
run_args += ['--access-logformat', str(args.access_logformat)]
if args.daemon:
run_args += ['--daemon']
if ssl_cert:
run_args += ['--certfile', ssl_cert, '--keyfile', ssl_key]
run_args += ["airflow.www.app:cached_app()"]
gunicorn_master_proc = None
def kill_proc(signum, _):
log.info("Received signal: %s. Closing gunicorn.", signum)
gunicorn_master_proc.terminate()
with suppress(TimeoutError):
gunicorn_master_proc.wait(timeout=30)
if gunicorn_master_proc.poll() is not None:
gunicorn_master_proc.kill()
sys.exit(0)
def monitor_gunicorn(gunicorn_master_pid: int):
# Register signal handlers
signal.signal(signal.SIGINT, kill_proc)
signal.signal(signal.SIGTERM, kill_proc)
# These run forever until SIG{INT, TERM, KILL, ...} signal is sent
GunicornMonitor(
gunicorn_master_pid=gunicorn_master_pid,
num_workers_expected=num_workers,
master_timeout=conf.getint('webserver',
'web_server_master_timeout'),
worker_refresh_interval=conf.getint('webserver',
'worker_refresh_interval',
fallback=30),
worker_refresh_batch_size=conf.getint(
'webserver', 'worker_refresh_batch_size', fallback=1),
reload_on_plugin_change=conf.getboolean(
'webserver', 'reload_on_plugin_change', fallback=False),
).start()
if args.daemon:
# This makes possible errors get reported before daemonization
os.environ['SKIP_DAGS_PARSING'] = 'True'
app = create_app(None)
os.environ.pop('SKIP_DAGS_PARSING')
handle = setup_logging(log_file)
base, ext = os.path.splitext(pid_file)
with open(stdout, 'w+') as stdout, open(stderr, 'w+') as stderr:
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(f"{base}-monitor{ext}", -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
subprocess.Popen(run_args, close_fds=True)
# Reading pid of gunicorn master as it will be different that
# the one of process spawned above.
while True:
sleep(0.1)
gunicorn_master_proc_pid = read_pid_from_pidfile(
pid_file)
if gunicorn_master_proc_pid:
break
# Run Gunicorn monitor
gunicorn_master_proc = psutil.Process(
gunicorn_master_proc_pid)
monitor_gunicorn(gunicorn_master_proc.pid)
else:
with subprocess.Popen(run_args,
close_fds=True) as gunicorn_master_proc:
monitor_gunicorn(gunicorn_master_proc.pid)
def worker(args):
"""Starts Airflow Celery worker"""
if not settings.validate_session():
print("Worker exiting... database connection precheck failed! ")
sys.exit(1)
autoscale = args.autoscale
skip_serve_logs = args.skip_serve_logs
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
# Setup locations
pid_file_path, stdout, stderr, log_file = setup_locations(
process=WORKER_PROCESS_NAME,
pid=args.pid,
stdout=args.stdout,
stderr=args.stderr,
log=args.log_file,
)
# Setup Celery worker
worker_instance = worker_bin.worker(app=celery_app)
options = {
'optimization': 'fair',
'O': 'fair',
'queues': args.queues,
'concurrency': args.concurrency,
'autoscale': autoscale,
'hostname': args.celery_hostname,
'loglevel': conf.get('logging', 'LOGGING_LEVEL'),
'pidfile': pid_file_path,
}
if conf.has_option("celery", "pool"):
options["pool"] = conf.get("celery", "pool")
if args.daemon:
# Run Celery worker as daemon
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
ctx = daemon.DaemonContext(
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
stdout.close()
stderr.close()
else:
# Run Celery worker in the same process
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
if sub_proc:
sub_proc.terminate()
def worker(args):
"""Starts Airflow Celery worker"""
if not settings.validate_session():
print("Worker exiting... database connection precheck failed! ")
sys.exit(1)
autoscale = args.autoscale
skip_serve_logs = args.skip_serve_logs
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
# Setup locations
pid_file_path, stdout, stderr, log_file = setup_locations(
process=WORKER_PROCESS_NAME,
pid=args.pid,
stdout=args.stdout,
stderr=args.stderr,
log=args.log_file,
)
# Setup Celery worker
worker_instance = worker_bin.worker(app=celery_app)
options = {
'optimization': 'fair',
'O': 'fair',
'queues': args.queues,
'concurrency': args.concurrency,
'autoscale': autoscale,
'hostname': args.celery_hostname,
'loglevel': conf.get('logging', 'LOGGING_LEVEL'),
'pidfile': pid_file_path,
}
if conf.has_option("celery", "pool"):
pool = conf.get("celery", "pool")
options["pool"] = pool
# Celery pools of type eventlet and gevent use greenlets, which
# requires monkey patching the app:
# https://eventlet.net/doc/patching.html#monkey-patch
# Otherwise task instances hang on the workers and are never
# executed.
maybe_patch_concurrency(['-P', pool])
if args.daemon:
# Run Celery worker as daemon
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
if args.umask:
umask = args.umask
ctx = daemon.DaemonContext(
files_preserve=[handle],
umask=int(umask, 8),
stdout=stdout,
stderr=stderr,
)
with ctx:
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
stdout.close()
stderr.close()
else:
# Run Celery worker in the same process
sub_proc = _serve_logs(skip_serve_logs)
worker_instance.run(**options)
if sub_proc:
sub_proc.terminate()
def worker(args):
"""Starts Airflow Celery worker"""
if not settings.validate_session():
raise SystemExit(
"Worker exiting, database connection precheck failed.")
autoscale = args.autoscale
skip_serve_logs = args.skip_serve_logs
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
# Setup locations
pid_file_path, stdout, stderr, log_file = setup_locations(
process=WORKER_PROCESS_NAME,
pid=args.pid,
stdout=args.stdout,
stderr=args.stderr,
log=args.log_file,
)
if hasattr(celery_app.backend, 'ResultSession'):
# Pre-create the database tables now, otherwise SQLA via Celery has a
# race condition where one of the subprocesses can die with "Table
# already exists" error, because SQLA checks for which tables exist,
# then issues a CREATE TABLE, rather than doing CREATE TABLE IF NOT
# EXISTS
try:
session = celery_app.backend.ResultSession()
session.close()
except sqlalchemy.exc.IntegrityError:
# At least on postgres, trying to create a table that already exist
# gives a unique constraint violation or the
# "pg_type_typname_nsp_index" table. If this happens we can ignore
# it, we raced to create the tables and lost.
pass
# Setup Celery worker
options = [
'worker',
'-O',
'fair',
'--queues',
args.queues,
'--concurrency',
args.concurrency,
'--hostname',
args.celery_hostname,
'--loglevel',
conf.get('logging', 'LOGGING_LEVEL'),
'--pidfile',
pid_file_path,
]
if autoscale:
options.extend(['--autoscale', autoscale])
if args.without_mingle:
options.append('--without-mingle')
if args.without_gossip:
options.append('--without-gossip')
if conf.has_option("celery", "pool"):
pool = conf.get("celery", "pool")
options.extend(["--pool", pool])
# Celery pools of type eventlet and gevent use greenlets, which
# requires monkey patching the app:
# https://eventlet.net/doc/patching.html#monkey-patch
# Otherwise task instances hang on the workers and are never
# executed.
maybe_patch_concurrency(['-P', pool])
if args.daemon:
# Run Celery worker as daemon
handle = setup_logging(log_file)
with open(stdout, 'w+') as stdout_handle, open(stderr,
'w+') as stderr_handle:
if args.umask:
umask = args.umask
ctx = daemon.DaemonContext(
files_preserve=[handle],
umask=int(umask, 8),
stdout=stdout_handle,
stderr=stderr_handle,
)
with ctx:
sub_proc = _serve_logs(skip_serve_logs)
celery_app.worker_main(options)
else:
# Run Celery worker in the same process
sub_proc = _serve_logs(skip_serve_logs)
celery_app.worker_main(options)
if sub_proc:
sub_proc.terminate()
def worker(args):
"""Starts Airflow Celery worker"""
env = os.environ.copy()
env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME
if not settings.validate_session():
print("Worker exiting... database connection precheck failed! ")
sys.exit(1)
# Celery worker
from airflow.executors.celery_executor import app as celery_app
from celery.bin import worker # pylint: disable=redefined-outer-name
autoscale = args.autoscale
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
worker = worker.worker(app=celery_app) # pylint: disable=redefined-outer-name
options = {
'optimization': 'fair',
'O': 'fair',
'queues': args.queues,
'concurrency': args.concurrency,
'autoscale': autoscale,
'hostname': args.celery_hostname,
'loglevel': conf.get('core', 'LOGGING_LEVEL'),
}
if conf.has_option("celery", "pool"):
options["pool"] = conf.get("celery", "pool")
if args.daemon:
pid, stdout, stderr, log_file = setup_locations("worker",
args.pid,
args.stdout,
args.stderr,
args.log_file)
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True)
worker.run(**options)
sub_proc.kill()
stdout.close()
stderr.close()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True)
worker.run(**options)
sub_proc.kill()