def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict["core"]["unit_test_mode"], "True")
# test env vars
self.assertEqual(cfg_dict["testsection"]["testkey"], "< hidden >")
# test defaults
conf.remove_option("core", "load_examples")
cfg_dict = conf.as_dict()
self.assertEqual(cfg_dict["core"]["load_examples"], "True")
# test display_source
cfg_dict = conf.as_dict(display_source=True)
self.assertEqual(cfg_dict["core"]["unit_test_mode"][1], "airflow.cfg")
self.assertEqual(cfg_dict["core"]["load_examples"][1], "default")
self.assertEqual(cfg_dict["testsection"]["testkey"], ("< hidden >", "env var"))
# test display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True)
self.assertEqual(cfg_dict["testsection"]["testkey"], "testvalue")
# test display_source and display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
self.assertEqual(cfg_dict["testsection"]["testkey"], ("testvalue", "env var"))
def test_conf_as_dict_sensitive(self):
# test display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True)
self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
self.assertEqual(cfg_dict['testsection']['testpercent'], 'with%percent')
# test display_source and display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
self.assertEqual(
cfg_dict['testsection']['testkey'], ('testvalue', 'env var'))
def test_conf_as_dict_sensitive(self):
# test display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True)
self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
self.assertEqual(cfg_dict['testsection']['testpercent'], 'with%percent')
# test display_source and display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
self.assertEqual(
cfg_dict['testsection']['testkey'], ('testvalue', 'env var'))
def get_config_with_source(include_default: bool = False) -> str:
"""Return configuration along with source for each option."""
config_dict = conf.as_dict(display_source=True)
with io.StringIO() as buf, redirect_stdout(buf):
for section, options in config_dict.items():
if not include_default:
options = {
key: (value, source)
for key, (value, source) in options.items()
if source != "default"
}
# Print the section only when there are options after filtering
if options:
print(f"[{section}]")
for key, (value, source) in options.items():
print(f"{key} = {value} [{source}]")
print()
code = buf.getvalue()
if is_terminal_support_colors():
code = pygments.highlight(code=code,
formatter=get_terminal_formatter(),
lexer=IniLexer())
return code
def test_conf_as_dict_exclude_env(self):
# test display_sensitive
cfg_dict = conf.as_dict(include_env=False, display_sensitive=True)
# Since testsection is only created from env vars, it shouldn't be
# present at all if we don't ask for env vars to be included.
self.assertNotIn('testsection', cfg_dict)
def tmp_configuration_copy(chmod=0o600, include_env=True, include_cmds=True):
"""
Returns a path for a temporary file including a full copy of the configuration
settings.
:param include_env: Should the value of configuration from ``AIRFLOW__``
environment variables be included or not
:type include_env: bool
:param include_cmds: Should the result of calling any *_cmd config be
set (True, default), or should the _cmd options be left as the
command to run (False)
:type include_cmds: bool
:return: a path to a temporary file
"""
cfg_dict = conf.as_dict(display_sensitive=True,
raw=True,
include_cmds=include_cmds,
include_env=include_env)
temp_fd, cfg_path = mkstemp()
with os.fdopen(temp_fd, 'w') as temp_file:
# Set the permissions before we write anything to it.
if chmod is not None:
os.fchmod(temp_fd, chmod)
json.dump(cfg_dict, temp_file)
return cfg_path
def test_conf_as_dict_source(self):
# test display_source
cfg_dict = conf.as_dict(display_source=True)
self.assertEqual(
cfg_dict['core']['load_examples'][1], 'airflow.cfg')
self.assertEqual(
cfg_dict['testsection']['testkey'], ('< hidden >', 'env var'))
def test_conf_as_dict_source(self):
# test display_source
cfg_dict = conf.as_dict(display_source=True)
self.assertEqual(
cfg_dict['core']['load_examples'][1], 'airflow.cfg')
self.assertEqual(
cfg_dict['testsection']['testkey'], ('< hidden >', 'env var'))
def test_conf_as_dict_raw(self):
# test display_sensitive
cfg_dict = conf.as_dict(raw=True, display_sensitive=True)
self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
# Values with '%' in them should be escaped
self.assertEqual(cfg_dict['testsection']['testpercent'], 'with%%percent')
self.assertEqual(cfg_dict['core']['percent'], 'with%%inside')
def test_conf_as_dict_raw(self):
# test display_sensitive
cfg_dict = conf.as_dict(raw=True, display_sensitive=True)
self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
# Values with '%' in them should be escaped
self.assertEqual(cfg_dict['testsection']['testpercent'], 'with%%percent')
self.assertEqual(cfg_dict['core']['percent'], 'with%%inside')
def test_config_as_dict(self):
"""Test that getting config as dict works even if
environment has non-legal env vars"""
with unittest.mock.patch.dict('os.environ'):
os.environ['AIRFLOW__VAR__broken'] = "not_ok"
asdict = conf.as_dict(raw=True, display_sensitive=True)
assert asdict.get('VAR') is None
assert asdict['testsection']['testkey'] == 'testvalue'
def show_config(args):
"""Show current application configuration"""
config = conf.as_dict(display_sensitive=True, raw=True)
for section_key, parameters in sorted(config.items()):
print(f"[{section_key}]")
for parameter_key, value in sorted(parameters.items()):
print(f"{parameter_key}={value}")
print()
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
self.assertEqual(cfg_dict['core']['percent'], 'with%inside')
# test env vars
self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
self.assertEqual(cfg_dict['core']['percent'], 'with%inside')
# test env vars
self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
assert cfg_dict['core']['unit_test_mode'] == 'True'
assert cfg_dict['core']['percent'] == 'with%inside'
# test env vars
assert cfg_dict['testsection']['testkey'] == '< hidden >'
assert cfg_dict['kubernetes_environment_variables']['AIRFLOW__TESTSECTION__TESTKEY'] == '< hidden >'
def __init__(self): # pylint: disable=too-many-statements
configuration_dict = conf.as_dict(display_sensitive=True)
self.core_configuration = configuration_dict['core']
self.airflow_home = settings.AIRFLOW_HOME
self.dags_folder = conf.get(self.core_section, 'dags_folder')
self.parallelism = conf.getint(self.core_section, 'parallelism')
self.pod_template_file = conf.get(self.kubernetes_section,
'pod_template_file',
fallback=None)
self.delete_worker_pods = conf.getboolean(self.kubernetes_section,
'delete_worker_pods')
self.delete_worker_pods_on_failure = conf.getboolean(
self.kubernetes_section, 'delete_worker_pods_on_failure')
self.worker_pods_creation_batch_size = conf.getint(
self.kubernetes_section, 'worker_pods_creation_batch_size')
self.worker_container_repository = conf.get(
self.kubernetes_section, 'worker_container_repository')
self.worker_container_tag = conf.get(self.kubernetes_section,
'worker_container_tag')
self.kube_image = f'{self.worker_container_repository}:{self.worker_container_tag}'
# The Kubernetes Namespace in which the Scheduler and Webserver reside. Note
# that if your
# cluster has RBAC enabled, your scheduler may need service account permissions to
# create, watch, get, and delete pods in this namespace.
self.kube_namespace = conf.get(self.kubernetes_section, 'namespace')
self.multi_namespace_mode = conf.getboolean(self.kubernetes_section,
'multi_namespace_mode')
# The Kubernetes Namespace in which pods will be created by the executor. Note
# that if your
# cluster has RBAC enabled, your workers may need service account permissions to
# interact with cluster components.
self.executor_namespace = conf.get(self.kubernetes_section,
'namespace')
kube_client_request_args = conf.get(self.kubernetes_section,
'kube_client_request_args')
if kube_client_request_args:
self.kube_client_request_args = json.loads(
kube_client_request_args)
if self.kube_client_request_args['_request_timeout'] and \
isinstance(self.kube_client_request_args['_request_timeout'], list):
self.kube_client_request_args['_request_timeout'] = \
tuple(self.kube_client_request_args['_request_timeout'])
else:
self.kube_client_request_args = {}
delete_option_kwargs = conf.get(self.kubernetes_section,
'delete_option_kwargs')
if delete_option_kwargs:
self.delete_option_kwargs = json.loads(delete_option_kwargs)
else:
self.delete_option_kwargs = {}
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
self.assertEqual(cfg_dict['core']['percent'], 'with%inside')
# test env vars
self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
self.assertEqual(
cfg_dict['kubernetes_environment_variables']
['airflow__testsection__testkey'], '< hidden >')
def _get_scratch_base_path() -> str:
dct = airflow_conf.as_dict(display_sensitive=True)['connections']
if 'WORKFLOW_SCRATCH' in dct:
scratch_path = dct['WORKFLOW_SCRATCH']
elif 'workflow_scratch' in dct:
# support for lower case is necessary setting the scratch path via the
# environment variable AIRFLOW__CONNECTIONS__WORKFLOW_SCRATCH
scratch_path = dct['workflow_scratch']
else:
raise KeyError('WORKFLOW_SCRATCH') # preserve original code behavior
scratch_path = scratch_path.strip("'").strip(
'"') # remove quotes that may be on the string
return scratch_path
def map_queue_name(raw_queue_name: str) -> str:
"""
If the configuration contains QUEUE_NAME_TEMPLATE, use it to customize the
provided queue name. This allows job separation under Celery.
"""
conf_dict = airflow_conf.as_dict()
if 'QUEUE_NAME_TEMPLATE' in conf_dict.get('connections', {}):
template = conf_dict['connections']['QUEUE_NAME_TEMPLATE']
template = template.strip("'").strip(
'"') # remove quotes that may be on the config string
rslt = template.format(raw_queue_name)
return rslt
else:
return raw_queue_name
def test_conf_as_dict(self):
os.environ['AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__AIRFLOW__TESTSECTION__TESTKEY'] = 'nested'
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
self.assertEqual(cfg_dict['core']['percent'], 'with%inside')
# test env vars
self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
self.assertEqual(
cfg_dict['kubernetes_environment_variables']['AIRFLOW__TESTSECTION__TESTKEY'],
'< hidden >')
del os.environ['AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__AIRFLOW__TESTSECTION__TESTKEY']
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
# test env vars
self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
# test display_source
cfg_dict = conf.as_dict(display_source=True)
self.assertEqual(
cfg_dict['core']['load_examples'][1], 'airflow config')
self.assertEqual(
cfg_dict['testsection']['testkey'], ('< hidden >', 'env var'))
# test display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True)
self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
# test display_source and display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
self.assertEqual(
cfg_dict['testsection']['testkey'], ('testvalue', 'env var'))
def get_config() -> Response:
"""Get current configuration."""
serializer = {
'text/plain': _config_to_text,
'application/json': _config_to_json,
}
response_types = serializer.keys()
return_type = request.accept_mimetypes.best_match(response_types)
conf_dict = conf.as_dict(display_source=False, display_sensitive=True)
config = _conf_dict_to_config(conf_dict)
if return_type not in serializer:
return Response(status=406)
else:
config_text = serializer[return_type](config)
return Response(config_text, headers={'Content-Type': return_type})
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
# test env vars
self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
# test display_source
cfg_dict = conf.as_dict(display_source=True)
self.assertEqual(
cfg_dict['core']['load_examples'][1], 'airflow config')
self.assertEqual(
cfg_dict['testsection']['testkey'], ('< hidden >', 'env var'))
# test display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True)
self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
# test display_source and display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
self.assertEqual(
cfg_dict['testsection']['testkey'], ('testvalue', 'env var'))
def tmp_configuration_copy(chmod=0o600):
"""
Returns a path for a temporary file including a full copy of the configuration
settings.
:return: a path to a temporary file
"""
cfg_dict = conf.as_dict(display_sensitive=True, raw=True)
temp_fd, cfg_path = mkstemp()
with os.fdopen(temp_fd, 'w') as temp_file:
# Set the permissions before we write anything to it.
if chmod is not None:
os.fchmod(temp_fd, chmod)
json.dump(cfg_dict, temp_file)
return cfg_path
def get_config() -> Response:
"""Get current configuration."""
serializer = {
'text/plain': _config_to_text,
'application/json': _config_to_json,
}
return_type = request.accept_mimetypes.best_match(serializer.keys())
if return_type not in serializer:
return Response(status=406)
elif conf.getboolean("webserver", "expose_config"):
conf_dict = conf.as_dict(display_source=False, display_sensitive=True)
config = _conf_dict_to_config(conf_dict)
config_text = serializer[return_type](config)
return Response(config_text, headers={'Content-Type': return_type})
else:
raise PermissionDenied(detail=(
'Your Airflow administrator chose not to expose the configuration, most likely for security'
' reasons.'))
def config(section, key):
dct = airflow_conf.as_dict(display_sensitive=True)
if section in dct:
if key in dct[section]:
rslt = dct[section][key]
elif key.lower() in dct[section]:
rslt = dct[section][key.lower()]
elif key.upper() in dct[section]:
rslt = dct[section][key.upper()]
else:
raise AirflowConfigException('No config entry for [{}] {}'.format(
section, key))
# airflow config reader leaves quotes, which we want to strip
for qc in ['"', "'"]:
if rslt.startswith(qc) and rslt.endswith(qc):
rslt = rslt.strip(qc)
return rslt
else:
raise AirflowConfigException('No config section [{}]'.format(section))
def check_config():
# Check for needed configuration elements, since it's better to fail early
dct = airflow_conf.as_dict(display_sensitive=True)
failed = 0
for elt in NEEDED_ENV_VARS:
if elt not in os.environ:
LOGGER.error('The environment variable {} is not set'.format(elt))
failed += 1
for key in NEEDED_CONFIG_SECTIONS + [a for a, b in NEEDED_CONFIGS]:
if not (key in dct or key.upper() in dct):
LOGGER.error(
'The configuration section {} does not exist'.format(key))
failed += 1
for key1, key2 in NEEDED_CONFIGS:
sub_dct = dct[key1] if key1 in dct else dct[key1.upper()]
if not (key2 in sub_dct or key2.upper() in sub_dct):
LOGGER.error(
'The configuration parameter [{}] {} does not exist'.format(
key1, key2))
failed += 1
assert failed == 0, 'ingest-pipeline plugin found {} configuration errors'.format(
failed)
def __init__(self):
configuration_dict = conf.as_dict(display_sensitive=True)
self.core_configuration = configuration_dict[self.core_section]
self.airflow_home = AIRFLOW_HOME
self.dags_folder = conf.get(self.core_section, 'dags_folder')
self.parallelism = conf.getint(self.core_section, 'parallelism')
self.pod_template_file = conf.get(self.kubernetes_section,
'pod_template_file',
fallback=None)
self.delete_worker_pods = conf.getboolean(self.kubernetes_section,
'delete_worker_pods')
self.delete_worker_pods_on_failure = conf.getboolean(
self.kubernetes_section, 'delete_worker_pods_on_failure')
self.worker_pods_creation_batch_size = conf.getint(
self.kubernetes_section, 'worker_pods_creation_batch_size')
self.worker_container_repository = conf.get(
self.kubernetes_section, 'worker_container_repository')
self.worker_container_tag = conf.get(self.kubernetes_section,
'worker_container_tag')
if self.worker_container_repository and self.worker_container_tag:
self.kube_image = f'{self.worker_container_repository}:{self.worker_container_tag}'
else:
self.kube_image = None
# The Kubernetes Namespace in which the Scheduler and Webserver reside. Note
# that if your
# cluster has RBAC enabled, your scheduler may need service account permissions to
# create, watch, get, and delete pods in this namespace.
self.kube_namespace = conf.get(self.kubernetes_section, 'namespace')
self.multi_namespace_mode = conf.getboolean(self.kubernetes_section,
'multi_namespace_mode')
# The Kubernetes Namespace in which pods will be created by the executor. Note
# that if your
# cluster has RBAC enabled, your workers may need service account permissions to
# interact with cluster components.
self.executor_namespace = conf.get(self.kubernetes_section,
'namespace')
self.worker_pods_pending_timeout = conf.getint(
self.kubernetes_section, 'worker_pods_pending_timeout')
self.worker_pods_pending_timeout_check_interval = conf.getint(
self.kubernetes_section,
'worker_pods_pending_timeout_check_interval')
self.worker_pods_pending_timeout_batch_size = conf.getint(
self.kubernetes_section, 'worker_pods_pending_timeout_batch_size')
self.worker_pods_queued_check_interval = conf.getint(
self.kubernetes_section, 'worker_pods_queued_check_interval')
self.kube_client_request_args = conf.getjson(
self.kubernetes_section, 'kube_client_request_args', fallback={})
if not isinstance(self.kube_client_request_args, dict):
raise AirflowConfigException(
f"[{self.kubernetes_section}] 'kube_client_request_args' expected a JSON dict, got "
+ type(self.kube_client_request_args).__name__)
if self.kube_client_request_args:
if '_request_timeout' in self.kube_client_request_args and isinstance(
self.kube_client_request_args['_request_timeout'], list):
self.kube_client_request_args['_request_timeout'] = tuple(
self.kube_client_request_args['_request_timeout'])
self.delete_option_kwargs = conf.getjson(self.kubernetes_section,
'delete_option_kwargs',
fallback={})
if not isinstance(self.delete_option_kwargs, dict):
raise AirflowConfigException(
f"[{self.kubernetes_section}] 'delete_option_kwargs' expected a JSON dict, got "
+ type(self.delete_option_kwargs).__name__)
def api_admin_view3(self):
LOGGER.info('In APIAdminView1.api_admin_view3')
return show_template('show_config.html',
title='Airflow Config',
dict_of_dicts=airflow_conf.as_dict())
user_defined_macros={'tmp_dir_path': utils.get_tmp_dir_path}) as dag:
prev = dag
for idx, uuid in enumerate(UUIDS_TO_RESET):
this_t = PythonOperator(
task_id=f'set_dataset_new_{idx}',
python_callable=utils.pythonop_set_dataset_state,
provide_context=True,
op_kwargs={
'dataset_uuid_callable':
uuid_fun,
'http_conn_id':
'ingest_api_connection',
'endpoint':
'/datasets/status',
'ds_state':
'New',
'message':
'Resetting state to NEW',
'crypt_auth_tok':
utils.encrypt_tok(airflow_conf.as_dict()['connections']
['APP_CLIENT_SECRET']).decode(),
'uuid':
uuid
})
prev >> this_t
prev = this_t
def get_process_strings():
dct = airflow_conf.as_dict()
psl = [s.upper() for s in dct['ingest_map']] if 'ingest_map' in dct else []
return HubmapApiResponse.success({'process_strings': psl})
def encrypt_tok(cleartext_tok: str) -> bytes:
key = airflow_conf.as_dict(display_sensitive=True)['core']['fernet_key']
fernet = Fernet(key.encode())
return fernet.encrypt(cleartext_tok.encode())
def decrypt_tok(crypt_tok: bytes) -> str:
key = airflow_conf.as_dict(display_sensitive=True)['core']['fernet_key']
fernet = Fernet(key.encode())
return fernet.decrypt(crypt_tok).decode()
def __init__(self): # pylint: disable=too-many-statements
configuration_dict = conf.as_dict(display_sensitive=True)
self.core_configuration = configuration_dict['core']
self.kube_secrets = configuration_dict.get('kubernetes_secrets', {})
self.kube_env_vars = configuration_dict.get(
'kubernetes_environment_variables', {})
self.env_from_configmap_ref = conf.get(self.kubernetes_section,
'env_from_configmap_ref')
self.env_from_secret_ref = conf.get(self.kubernetes_section,
'env_from_secret_ref')
self.airflow_home = settings.AIRFLOW_HOME
self.dags_folder = conf.get(self.core_section, 'dags_folder')
self.parallelism = conf.getint(self.core_section, 'parallelism')
self.worker_container_repository = conf.get(
self.kubernetes_section, 'worker_container_repository')
self.worker_container_tag = conf.get(self.kubernetes_section,
'worker_container_tag')
self.kube_image = '{}:{}'.format(self.worker_container_repository,
self.worker_container_tag)
self.kube_image_pull_policy = conf.get(
self.kubernetes_section, "worker_container_image_pull_policy")
self.kube_node_selectors = configuration_dict.get(
'kubernetes_node_selectors', {})
self.pod_template_file = conf.get(self.kubernetes_section,
'pod_template_file',
fallback=None)
kube_worker_annotations = conf.get(self.kubernetes_section,
'worker_annotations')
if kube_worker_annotations:
self.kube_annotations = json.loads(kube_worker_annotations)
else:
self.kube_annotations = None
self.kube_labels = configuration_dict.get('kubernetes_labels', {})
self.delete_worker_pods = conf.getboolean(self.kubernetes_section,
'delete_worker_pods')
self.delete_worker_pods_on_failure = conf.getboolean(
self.kubernetes_section, 'delete_worker_pods_on_failure')
self.worker_pods_creation_batch_size = conf.getint(
self.kubernetes_section, 'worker_pods_creation_batch_size')
self.worker_service_account_name = conf.get(
self.kubernetes_section, 'worker_service_account_name')
self.image_pull_secrets = conf.get(self.kubernetes_section,
'image_pull_secrets')
# NOTE: user can build the dags into the docker image directly,
# this will set to True if so
self.dags_in_image = conf.getboolean(self.kubernetes_section,
'dags_in_image')
# Run as user for pod security context
self.worker_run_as_user = self._get_security_context_val('run_as_user')
self.worker_fs_group = self._get_security_context_val('fs_group')
kube_worker_resources = conf.get(self.kubernetes_section,
'worker_resources')
if kube_worker_resources:
self.worker_resources = json.loads(kube_worker_resources)
else:
self.worker_resources = None
# NOTE: `git_repo` and `git_branch` must be specified together as a pair
# The http URL of the git repository to clone from
self.git_repo = conf.get(self.kubernetes_section, 'git_repo')
# The branch of the repository to be checked out
self.git_branch = conf.get(self.kubernetes_section, 'git_branch')
# Optionally, the directory in the git repository containing the dags
self.git_subpath = conf.get(self.kubernetes_section, 'git_subpath')
# Optionally, the root directory for git operations
self.git_sync_root = conf.get(self.kubernetes_section, 'git_sync_root')
# Optionally, the name at which to publish the checked-out files under --root
self.git_sync_dest = conf.get(self.kubernetes_section, 'git_sync_dest')
# Optionally, the tag or hash to checkout
self.git_sync_rev = conf.get(self.kubernetes_section, 'git_sync_rev')
# Optionally, if git_dags_folder_mount_point is set the worker will use
# {git_dags_folder_mount_point}/{git_sync_dest}/{git_subpath} as dags_folder
self.git_dags_folder_mount_point = conf.get(
self.kubernetes_section, 'git_dags_folder_mount_point')
# Optionally a user may supply a (`git_user` AND `git_password`) OR
# (`git_ssh_key_secret_name` AND `git_ssh_key_secret_key`) for private repositories
self.git_user = conf.get(self.kubernetes_section, 'git_user')
self.git_password = conf.get(self.kubernetes_section, 'git_password')
self.git_ssh_key_secret_name = conf.get(self.kubernetes_section,
'git_ssh_key_secret_name')
self.git_ssh_known_hosts_configmap_name = conf.get(
self.kubernetes_section, 'git_ssh_known_hosts_configmap_name')
self.git_sync_credentials_secret = conf.get(
self.kubernetes_section, 'git_sync_credentials_secret')
# NOTE: The user may optionally use a volume claim to mount a PV containing
# DAGs directly
self.dags_volume_claim = conf.get(self.kubernetes_section,
'dags_volume_claim')
self.dags_volume_mount_point = conf.get(self.kubernetes_section,
'dags_volume_mount_point')
# This prop may optionally be set for PV Claims and is used to write logs
self.logs_volume_claim = conf.get(self.kubernetes_section,
'logs_volume_claim')
# This prop may optionally be set for PV Claims and is used to locate DAGs
# on a SubPath
self.dags_volume_subpath = conf.get(self.kubernetes_section,
'dags_volume_subpath')
# This prop may optionally be set for PV Claims and is used to locate logs
# on a SubPath
self.logs_volume_subpath = conf.get(self.kubernetes_section,
'logs_volume_subpath')
# Optionally, hostPath volume containing DAGs
self.dags_volume_host = conf.get(self.kubernetes_section,
'dags_volume_host')
# Optionally, write logs to a hostPath Volume
self.logs_volume_host = conf.get(self.kubernetes_section,
'logs_volume_host')
# This prop may optionally be set for PV Claims and is used to write logs
self.base_log_folder = conf.get(self.logging_section,
'base_log_folder')
# The Kubernetes Namespace in which the Scheduler and Webserver reside. Note
# that if your
# cluster has RBAC enabled, your scheduler may need service account permissions to
# create, watch, get, and delete pods in this namespace.
self.kube_namespace = conf.get(self.kubernetes_section, 'namespace')
# The Kubernetes Namespace in which pods will be created by the executor. Note
# that if your
# cluster has RBAC enabled, your workers may need service account permissions to
# interact with cluster components.
self.executor_namespace = conf.get(self.kubernetes_section,
'namespace')
# If the user is using the git-sync container to clone their repository via git,
# allow them to specify repository, tag, and pod name for the init container.
self.git_sync_container_repository = conf.get(
self.kubernetes_section, 'git_sync_container_repository')
self.git_sync_container_tag = conf.get(self.kubernetes_section,
'git_sync_container_tag')
self.git_sync_container = '{}:{}'.format(
self.git_sync_container_repository, self.git_sync_container_tag)
self.git_sync_init_container_name = conf.get(
self.kubernetes_section, 'git_sync_init_container_name')
self.git_sync_run_as_user = self._get_security_context_val(
'git_sync_run_as_user')
# The worker pod may optionally have a valid Airflow config loaded via a
# configmap
self.airflow_configmap = conf.get(self.kubernetes_section,
'airflow_configmap')
# The worker pod may optionally have a valid Airflow local settings loaded via a
# configmap
self.airflow_local_settings_configmap = conf.get(
self.kubernetes_section, 'airflow_local_settings_configmap')
affinity_json = conf.get(self.kubernetes_section, 'affinity')
if affinity_json:
self.kube_affinity = json.loads(affinity_json)
else:
self.kube_affinity = None
tolerations_json = conf.get(self.kubernetes_section, 'tolerations')
if tolerations_json:
self.kube_tolerations = json.loads(tolerations_json)
else:
self.kube_tolerations = None
kube_client_request_args = conf.get(self.kubernetes_section,
'kube_client_request_args')
if kube_client_request_args:
self.kube_client_request_args = json.loads(
kube_client_request_args)
if self.kube_client_request_args['_request_timeout'] and \
isinstance(self.kube_client_request_args['_request_timeout'], list):
self.kube_client_request_args['_request_timeout'] = \
tuple(self.kube_client_request_args['_request_timeout'])
else:
self.kube_client_request_args = {}
self._validate()
delete_option_kwargs = conf.get(self.kubernetes_section,
'delete_option_kwargs')
if delete_option_kwargs:
self.delete_option_kwargs = json.loads(delete_option_kwargs)
else:
self.delete_option_kwargs = {}
def test_conf_as_dict_source(self):
# test display_source
cfg_dict = conf.as_dict(display_source=True)
assert cfg_dict['core']['load_examples'][1] == 'airflow.cfg'
assert cfg_dict['core']['load_default_connections'][1] == 'airflow.cfg'
assert cfg_dict['testsection']['testkey'] == ('< hidden >', 'env var')
