def __init__(self,
base_log_folder,
filename_template,
log_id_template,
end_of_log_mark,
write_stdout,
json_format,
json_fields,
host='localhost:9200',
es_kwargs=conf.getsection("elasticsearch_configs") or {}):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
super().__init__(base_log_folder, filename_template)
self.closed = False
self.log_id_template, self.log_id_jinja_template = \
parse_template_string(log_id_template)
self.client = elasticsearch.Elasticsearch([host], **es_kwargs)
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
self.write_stdout = write_stdout
self.json_format = json_format
self.json_fields = [label.strip() for label in json_fields.split(",")]
self.handler = None
def prepare_jinja_globals():
extra_globals = {
'server_timezone':
server_timezone,
'default_ui_timezone':
default_ui_timezone,
'hostname':
hosstname,
'navbar_color':
conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec':
conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset':
conf.getint('webserver', 'log_auto_tailing_offset', fallback=30),
'log_animation_speed':
conf.getint('webserver', 'log_animation_speed', fallback=1000),
'state_color_mapping':
STATE_COLORS,
'airflow_version':
airflow_version,
'git_version':
git_version,
'k8s_or_k8scelery_executor':
IS_K8S_OR_K8SCELERY_EXECUTOR,
}
if 'analytics_tool' in conf.getsection('webserver'):
extra_globals.update({
'analytics_tool':
conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id':
conf.get('webserver', 'ANALYTICS_ID'),
})
return extra_globals
def prepare_jinja_globals():
extra_globals = {
'server_timezone': server_timezone,
'default_ui_timezone': default_ui_timezone,
'hostname': hostname,
'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30),
'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000),
'state_color_mapping': STATE_COLORS,
'airflow_version': airflow_version,
'git_version': git_version,
'k8s_or_k8scelery_executor': IS_K8S_OR_K8SCELERY_EXECUTOR,
'rest_api_enabled': False,
}
backends = conf.get('api', 'auth_backends')
if len(backends) > 0 and backends[0] != 'airflow.api.auth.backend.deny_all':
extra_globals['rest_api_enabled'] = True
if 'analytics_tool' in conf.getsection('webserver'):
extra_globals.update(
{
'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id': conf.get('webserver', 'ANALYTICS_ID'),
}
)
return extra_globals
def __init__( # pylint: disable=too-many-arguments
self,
base_log_folder: str,
filename_template: str,
log_id_template: str,
end_of_log_mark: str,
write_stdout: bool,
json_format: bool,
json_fields: str,
host: str = "localhost:9200",
frontend: str = "localhost:5601",
es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"),
):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
es_kwargs = es_kwargs or {}
super().__init__(base_log_folder, filename_template)
self.closed = False
self.log_id_template, self.log_id_jinja_template = parse_template_string(
log_id_template)
self.client = elasticsearch.Elasticsearch([host], **es_kwargs)
self.frontend = frontend
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
self.write_stdout = write_stdout
self.json_format = json_format
self.json_fields = [label.strip() for label in json_fields.split(",")]
self.handler = None
self.context_set = False
def prepare_jinja_globals():
extra_globals = {
'server_timezone':
server_timezone,
'default_ui_timezone':
default_ui_timezone,
'hostname':
hosstname,
'navbar_color':
conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec':
conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset':
conf.getint('webserver', 'log_auto_tailing_offset', fallback=30),
'log_animation_speed':
conf.getint('webserver', 'log_animation_speed', fallback=1000),
}
if 'analytics_tool' in conf.getsection('webserver'):
extra_globals.update({
'analytics_tool':
conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id':
conf.get('webserver', 'ANALYTICS_ID'),
})
return extra_globals
def jinja_globals(): # pylint: disable=unused-variable
globals = {
'server_timezone':
server_timezone,
'default_ui_timezone':
default_ui_timezone,
'hostname':
socket.getfqdn() if conf.getboolean(
'webserver', 'EXPOSE_HOSTNAME', fallback=True) else
'redact',
'navbar_color':
conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec':
conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset':
conf.getint('webserver',
'log_auto_tailing_offset',
fallback=30),
'log_animation_speed':
conf.getint('webserver', 'log_animation_speed', fallback=1000)
}
if 'analytics_tool' in conf.getsection('webserver'):
globals.update({
'analytics_tool':
conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id':
conf.get('webserver', 'ANALYTICS_ID')
})
return globals
def test_broker_transport_options(self):
section_dict = conf.getsection("celery_broker_transport_options")
self.assertTrue(isinstance(section_dict['visibility_timeout'], int))
self.assertTrue(isinstance(section_dict['_test_only_bool'], bool))
self.assertTrue(isinstance(section_dict['_test_only_float'], float))
self.assertTrue(
isinstance(section_dict['_test_only_string'], six.string_types))
def test_broker_transport_options(self):
section_dict = conf.getsection("celery_broker_transport_options")
self.assertTrue(isinstance(section_dict['visibility_timeout'], int))
self.assertTrue(isinstance(section_dict['_test_only_bool'], bool))
self.assertTrue(isinstance(section_dict['_test_only_float'], float))
self.assertTrue(isinstance(section_dict['_test_only_string'], six.string_types))
def test_get_section_should_respect_cmd_env_variable(self):
with tempfile.NamedTemporaryFile(delete=False) as cmd_file:
cmd_file.write(b"#!/usr/bin/env bash\n")
cmd_file.write(b"echo -n difficult_unpredictable_cat_password\n")
cmd_file.flush()
os.chmod(cmd_file.name, 0o0555)
cmd_file.close()
with mock.patch.dict("os.environ", {"AIRFLOW__WEBSERVER__SECRET_KEY_CMD": cmd_file.name}):
content = conf.getsection("webserver")
os.unlink(cmd_file.name)
assert content["secret_key"] == "difficult_unpredictable_cat_password"
def test_client_with_config(self):
es_conf = dict(conf.getsection("elasticsearch_configs"))
expected_dict = {
"use_ssl": False,
"verify_certs": True,
}
self.assertDictEqual(es_conf, expected_dict)
# ensure creating with configs does not fail
ElasticsearchTaskHandler(self.local_log_location,
self.filename_template, self.log_id_template,
self.end_of_log_mark, self.write_stdout,
self.json_format, self.json_fields, es_conf)
def test_get_section_should_respect_cmd_env_variable(self):
with tempfile.NamedTemporaryFile(delete=False) as cmd_file:
cmd_file.write(b"#!/usr/bin/env bash\n")
cmd_file.write(b"echo -n difficult_unpredictable_cat_password\n")
cmd_file.flush()
os.chmod(cmd_file.name, 0o0555)
cmd_file.close()
with mock.patch.dict(
"os.environ", {"AIRFLOW__KUBERNETES__GIT_PASSWORD_CMD": cmd_file.name}
):
content = conf.getsection("kubernetes")
os.unlink(cmd_file.name)
self.assertEqual(content["git_password"], "difficult_unpredictable_cat_password")
def jinja_globals(): # pylint: disable=unused-variable
globals = {
'hostname': socket.getfqdn(),
'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'),
}
if 'analytics_tool' in conf.getsection('webserver'):
globals.update({
'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id': conf.get('webserver', 'ANALYTICS_ID')
})
return globals
def __init__(
self,
base_log_folder: str,
end_of_log_mark: str,
write_stdout: bool,
json_format: bool,
json_fields: str,
host_field: str = "host",
offset_field: str = "offset",
host: str = "localhost:9200",
frontend: str = "localhost:5601",
es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"),
*,
filename_template: Optional[str] = None,
log_id_template: Optional[str] = None,
):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
es_kwargs = es_kwargs or {}
super().__init__(base_log_folder, filename_template)
self.closed = False
self.client = elasticsearch.Elasticsearch(
[host], **es_kwargs) # type: ignore[attr-defined]
if USE_PER_RUN_LOG_ID and log_id_template is not None:
warnings.warn(
"Passing log_id_template to ElasticsearchTaskHandler is deprecated and has no effect",
DeprecationWarning,
)
self.log_id_template = log_id_template # Only used on Airflow < 2.3.2.
self.frontend = frontend
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
self.write_stdout = write_stdout
self.json_format = json_format
self.json_fields = [label.strip() for label in json_fields.split(",")]
self.host_field = host_field
self.offset_field = offset_field
self.context_set = False
self.formatter: logging.Formatter
self.handler: Union[logging.FileHandler,
logging.StreamHandler] # type: ignore[assignment]
def __init__(self):
"""Initialize the Sentry SDK."""
ignore_logger("airflow.task")
ignore_logger("airflow.jobs.backfill_job.BackfillJob")
executor_name = conf.get("core", "EXECUTOR")
sentry_flask = FlaskIntegration()
# LoggingIntegration is set by default.
integrations = [sentry_flask]
if executor_name == "CeleryExecutor":
from sentry_sdk.integrations.celery import CeleryIntegration
sentry_celery = CeleryIntegration()
integrations.append(sentry_celery)
dsn = None
sentry_config_opts = conf.getsection("sentry") or {}
if sentry_config_opts:
sentry_config_opts.pop("sentry_on")
old_way_dsn = sentry_config_opts.pop("sentry_dsn", None)
new_way_dsn = sentry_config_opts.pop("dsn", None)
# supported backward compatibility with old way dsn option
dsn = old_way_dsn or new_way_dsn
unsupported_options = self.UNSUPPORTED_SENTRY_OPTIONS.intersection(
sentry_config_opts.keys())
if unsupported_options:
log.warning(
"There are unsupported options in [sentry] section: %s",
", ".join(unsupported_options),
)
sentry_config_opts['before_send'] = conf.getimport(
'sentry', 'before_send', fallback=None)
if dsn:
sentry_sdk.init(dsn=dsn,
integrations=integrations,
**sentry_config_opts)
else:
# Setting up Sentry using environment variables.
log.debug("Defaulting to SENTRY_DSN in environment.")
sentry_sdk.init(integrations=integrations,
**sentry_config_opts)
def jinja_globals(): # pylint: disable=unused-variable
globals = {
'hostname': socket.getfqdn() if conf.getboolean(
'webserver',
'EXPOSE_HOSTNAME',
fallback=True) else 'redact',
'navbar_color': conf.get(
'webserver',
'NAVBAR_COLOR'),
}
if 'analytics_tool' in conf.getsection('webserver'):
globals.update({
'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id': conf.get('webserver', 'ANALYTICS_ID')
})
return globals
def __init__(
self,
base_log_folder: str,
filename_template: str,
log_id_template: str,
end_of_log_mark: str,
write_stdout: bool,
json_format: bool,
json_fields: str,
host_field: str = "host",
offset_field: str = "offset",
host: str = "localhost:9200",
frontend: str = "localhost:5601",
es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"),
):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
es_kwargs = es_kwargs or {}
super().__init__(base_log_folder, filename_template)
self.closed = False
self.client = elasticsearch.Elasticsearch(
[host], **es_kwargs) # type: ignore[attr-defined]
self.log_id_template = log_id_template
self.frontend = frontend
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
self.write_stdout = write_stdout
self.json_format = json_format
self.json_fields = [label.strip() for label in json_fields.split(",")]
self.host_field = host_field
self.offset_field = offset_field
self.context_set = False
self.formatter: logging.Formatter
self.handler: Union[logging.FileHandler,
logging.StreamHandler] # type: ignore[assignment]
def test_broker_transport_options(self):
section_dict = conf.getsection("celery_broker_transport_options")
assert isinstance(section_dict['visibility_timeout'], int)
assert isinstance(section_dict['_test_only_bool'], bool)
assert isinstance(section_dict['_test_only_float'], float)
assert isinstance(section_dict['_test_only_string'], str)
import logging
import ssl
from airflow.configuration import conf
from airflow.exceptions import AirflowConfigException, AirflowException
def _broker_supports_visibility_timeout(url):
return url.startswith("redis://") or url.startswith("sqs://")
log = logging.getLogger(__name__)
broker_url = conf.get('celery', 'BROKER_URL')
broker_transport_options = conf.getsection(
'celery_broker_transport_options') or {}
if 'visibility_timeout' not in broker_transport_options:
if _broker_supports_visibility_timeout(broker_url):
broker_transport_options['visibility_timeout'] = 21600
DEFAULT_CELERY_CONFIG = {
'accept_content': ['json'],
'event_serializer':
'json',
'worker_prefetch_multiplier':
conf.getint('celery', 'worker_prefetch_multiplier', fallback=1),
'task_acks_late':
True,
'task_default_queue':
conf.get('celery', 'DEFAULT_QUEUE'),
'task_default_exchange':