def __init__(self, base_log_folder, filename_template, log_id_template, end_of_log_mark, write_stdout, json_format, json_fields, host='localhost:9200', es_kwargs=conf.getsection("elasticsearch_configs") or {}): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ super().__init__(base_log_folder, filename_template) self.closed = False self.log_id_template, self.log_id_jinja_template = \ parse_template_string(log_id_template) self.client = elasticsearch.Elasticsearch([host], **es_kwargs) self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark self.write_stdout = write_stdout self.json_format = json_format self.json_fields = [label.strip() for label in json_fields.split(",")] self.handler = None
def prepare_jinja_globals(): extra_globals = { 'server_timezone': server_timezone, 'default_ui_timezone': default_ui_timezone, 'hostname': hosstname, 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), 'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2), 'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30), 'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000), 'state_color_mapping': STATE_COLORS, 'airflow_version': airflow_version, 'git_version': git_version, 'k8s_or_k8scelery_executor': IS_K8S_OR_K8SCELERY_EXECUTOR, } if 'analytics_tool' in conf.getsection('webserver'): extra_globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID'), }) return extra_globals
def prepare_jinja_globals(): extra_globals = { 'server_timezone': server_timezone, 'default_ui_timezone': default_ui_timezone, 'hostname': hostname, 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), 'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2), 'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30), 'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000), 'state_color_mapping': STATE_COLORS, 'airflow_version': airflow_version, 'git_version': git_version, 'k8s_or_k8scelery_executor': IS_K8S_OR_K8SCELERY_EXECUTOR, 'rest_api_enabled': False, } backends = conf.get('api', 'auth_backends') if len(backends) > 0 and backends[0] != 'airflow.api.auth.backend.deny_all': extra_globals['rest_api_enabled'] = True if 'analytics_tool' in conf.getsection('webserver'): extra_globals.update( { 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID'), } ) return extra_globals
def __init__( # pylint: disable=too-many-arguments self, base_log_folder: str, filename_template: str, log_id_template: str, end_of_log_mark: str, write_stdout: bool, json_format: bool, json_fields: str, host: str = "localhost:9200", frontend: str = "localhost:5601", es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"), ): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ es_kwargs = es_kwargs or {} super().__init__(base_log_folder, filename_template) self.closed = False self.log_id_template, self.log_id_jinja_template = parse_template_string( log_id_template) self.client = elasticsearch.Elasticsearch([host], **es_kwargs) self.frontend = frontend self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark self.write_stdout = write_stdout self.json_format = json_format self.json_fields = [label.strip() for label in json_fields.split(",")] self.handler = None self.context_set = False
def prepare_jinja_globals(): extra_globals = { 'server_timezone': server_timezone, 'default_ui_timezone': default_ui_timezone, 'hostname': hosstname, 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), 'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2), 'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30), 'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000), } if 'analytics_tool' in conf.getsection('webserver'): extra_globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID'), }) return extra_globals
def jinja_globals(): # pylint: disable=unused-variable globals = { 'server_timezone': server_timezone, 'default_ui_timezone': default_ui_timezone, 'hostname': socket.getfqdn() if conf.getboolean( 'webserver', 'EXPOSE_HOSTNAME', fallback=True) else 'redact', 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), 'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2), 'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30), 'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000) } if 'analytics_tool' in conf.getsection('webserver'): globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID') }) return globals
def test_broker_transport_options(self): section_dict = conf.getsection("celery_broker_transport_options") self.assertTrue(isinstance(section_dict['visibility_timeout'], int)) self.assertTrue(isinstance(section_dict['_test_only_bool'], bool)) self.assertTrue(isinstance(section_dict['_test_only_float'], float)) self.assertTrue( isinstance(section_dict['_test_only_string'], six.string_types))
def test_broker_transport_options(self): section_dict = conf.getsection("celery_broker_transport_options") self.assertTrue(isinstance(section_dict['visibility_timeout'], int)) self.assertTrue(isinstance(section_dict['_test_only_bool'], bool)) self.assertTrue(isinstance(section_dict['_test_only_float'], float)) self.assertTrue(isinstance(section_dict['_test_only_string'], six.string_types))
def test_get_section_should_respect_cmd_env_variable(self): with tempfile.NamedTemporaryFile(delete=False) as cmd_file: cmd_file.write(b"#!/usr/bin/env bash\n") cmd_file.write(b"echo -n difficult_unpredictable_cat_password\n") cmd_file.flush() os.chmod(cmd_file.name, 0o0555) cmd_file.close() with mock.patch.dict("os.environ", {"AIRFLOW__WEBSERVER__SECRET_KEY_CMD": cmd_file.name}): content = conf.getsection("webserver") os.unlink(cmd_file.name) assert content["secret_key"] == "difficult_unpredictable_cat_password"
def test_client_with_config(self): es_conf = dict(conf.getsection("elasticsearch_configs")) expected_dict = { "use_ssl": False, "verify_certs": True, } self.assertDictEqual(es_conf, expected_dict) # ensure creating with configs does not fail ElasticsearchTaskHandler(self.local_log_location, self.filename_template, self.log_id_template, self.end_of_log_mark, self.write_stdout, self.json_format, self.json_fields, es_conf)
def test_get_section_should_respect_cmd_env_variable(self): with tempfile.NamedTemporaryFile(delete=False) as cmd_file: cmd_file.write(b"#!/usr/bin/env bash\n") cmd_file.write(b"echo -n difficult_unpredictable_cat_password\n") cmd_file.flush() os.chmod(cmd_file.name, 0o0555) cmd_file.close() with mock.patch.dict( "os.environ", {"AIRFLOW__KUBERNETES__GIT_PASSWORD_CMD": cmd_file.name} ): content = conf.getsection("kubernetes") os.unlink(cmd_file.name) self.assertEqual(content["git_password"], "difficult_unpredictable_cat_password")
def jinja_globals(): # pylint: disable=unused-variable globals = { 'hostname': socket.getfqdn(), 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), } if 'analytics_tool' in conf.getsection('webserver'): globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID') }) return globals
def __init__( self, base_log_folder: str, end_of_log_mark: str, write_stdout: bool, json_format: bool, json_fields: str, host_field: str = "host", offset_field: str = "offset", host: str = "localhost:9200", frontend: str = "localhost:5601", es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"), *, filename_template: Optional[str] = None, log_id_template: Optional[str] = None, ): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ es_kwargs = es_kwargs or {} super().__init__(base_log_folder, filename_template) self.closed = False self.client = elasticsearch.Elasticsearch( [host], **es_kwargs) # type: ignore[attr-defined] if USE_PER_RUN_LOG_ID and log_id_template is not None: warnings.warn( "Passing log_id_template to ElasticsearchTaskHandler is deprecated and has no effect", DeprecationWarning, ) self.log_id_template = log_id_template # Only used on Airflow < 2.3.2. self.frontend = frontend self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark self.write_stdout = write_stdout self.json_format = json_format self.json_fields = [label.strip() for label in json_fields.split(",")] self.host_field = host_field self.offset_field = offset_field self.context_set = False self.formatter: logging.Formatter self.handler: Union[logging.FileHandler, logging.StreamHandler] # type: ignore[assignment]
def __init__(self): """Initialize the Sentry SDK.""" ignore_logger("airflow.task") ignore_logger("airflow.jobs.backfill_job.BackfillJob") executor_name = conf.get("core", "EXECUTOR") sentry_flask = FlaskIntegration() # LoggingIntegration is set by default. integrations = [sentry_flask] if executor_name == "CeleryExecutor": from sentry_sdk.integrations.celery import CeleryIntegration sentry_celery = CeleryIntegration() integrations.append(sentry_celery) dsn = None sentry_config_opts = conf.getsection("sentry") or {} if sentry_config_opts: sentry_config_opts.pop("sentry_on") old_way_dsn = sentry_config_opts.pop("sentry_dsn", None) new_way_dsn = sentry_config_opts.pop("dsn", None) # supported backward compatibility with old way dsn option dsn = old_way_dsn or new_way_dsn unsupported_options = self.UNSUPPORTED_SENTRY_OPTIONS.intersection( sentry_config_opts.keys()) if unsupported_options: log.warning( "There are unsupported options in [sentry] section: %s", ", ".join(unsupported_options), ) sentry_config_opts['before_send'] = conf.getimport( 'sentry', 'before_send', fallback=None) if dsn: sentry_sdk.init(dsn=dsn, integrations=integrations, **sentry_config_opts) else: # Setting up Sentry using environment variables. log.debug("Defaulting to SENTRY_DSN in environment.") sentry_sdk.init(integrations=integrations, **sentry_config_opts)
def jinja_globals(): # pylint: disable=unused-variable globals = { 'hostname': socket.getfqdn() if conf.getboolean( 'webserver', 'EXPOSE_HOSTNAME', fallback=True) else 'redact', 'navbar_color': conf.get( 'webserver', 'NAVBAR_COLOR'), } if 'analytics_tool' in conf.getsection('webserver'): globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID') }) return globals
def __init__( self, base_log_folder: str, filename_template: str, log_id_template: str, end_of_log_mark: str, write_stdout: bool, json_format: bool, json_fields: str, host_field: str = "host", offset_field: str = "offset", host: str = "localhost:9200", frontend: str = "localhost:5601", es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"), ): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ es_kwargs = es_kwargs or {} super().__init__(base_log_folder, filename_template) self.closed = False self.client = elasticsearch.Elasticsearch( [host], **es_kwargs) # type: ignore[attr-defined] self.log_id_template = log_id_template self.frontend = frontend self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark self.write_stdout = write_stdout self.json_format = json_format self.json_fields = [label.strip() for label in json_fields.split(",")] self.host_field = host_field self.offset_field = offset_field self.context_set = False self.formatter: logging.Formatter self.handler: Union[logging.FileHandler, logging.StreamHandler] # type: ignore[assignment]
def test_broker_transport_options(self): section_dict = conf.getsection("celery_broker_transport_options") assert isinstance(section_dict['visibility_timeout'], int) assert isinstance(section_dict['_test_only_bool'], bool) assert isinstance(section_dict['_test_only_float'], float) assert isinstance(section_dict['_test_only_string'], str)
import logging import ssl from airflow.configuration import conf from airflow.exceptions import AirflowConfigException, AirflowException def _broker_supports_visibility_timeout(url): return url.startswith("redis://") or url.startswith("sqs://") log = logging.getLogger(__name__) broker_url = conf.get('celery', 'BROKER_URL') broker_transport_options = conf.getsection( 'celery_broker_transport_options') or {} if 'visibility_timeout' not in broker_transport_options: if _broker_supports_visibility_timeout(broker_url): broker_transport_options['visibility_timeout'] = 21600 DEFAULT_CELERY_CONFIG = { 'accept_content': ['json'], 'event_serializer': 'json', 'worker_prefetch_multiplier': conf.getint('celery', 'worker_prefetch_multiplier', fallback=1), 'task_acks_late': True, 'task_default_queue': conf.get('celery', 'DEFAULT_QUEUE'), 'task_default_exchange':