def test_command_precedence(self):
test_config = '''[test]
key1 = hello
key2_cmd = printf cmd_result
key3 = airflow
key4_cmd = printf key4_result
'''
test_config_default = '''[test]
key1 = awesome
key2 = airflow
[another]
key6 = value6
'''
test_conf = AirflowConfigParser(
default_config=parameterized_config(test_config_default))
test_conf.read_string(test_config)
test_conf.sensitive_config_values = test_conf.sensitive_config_values | {
('test', 'key2'),
('test', 'key4'),
}
self.assertEqual('hello', test_conf.get('test', 'key1'))
self.assertEqual('cmd_result', test_conf.get('test', 'key2'))
self.assertEqual('airflow', test_conf.get('test', 'key3'))
self.assertEqual('key4_result', test_conf.get('test', 'key4'))
self.assertEqual('value6', test_conf.get('another', 'key6'))
self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb'))
self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb'))
self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb'))
self.assertEqual(True, test_conf.getboolean('another', 'key8_boolean', fallback='True'))
self.assertEqual(10, test_conf.getint('another', 'key8_int', fallback='10'))
self.assertEqual(1.0, test_conf.getfloat('another', 'key8_float', fallback='1'))
self.assertTrue(test_conf.has_option('test', 'key1'))
self.assertTrue(test_conf.has_option('test', 'key2'))
self.assertTrue(test_conf.has_option('test', 'key3'))
self.assertTrue(test_conf.has_option('test', 'key4'))
self.assertFalse(test_conf.has_option('test', 'key5'))
self.assertTrue(test_conf.has_option('another', 'key6'))
cfg_dict = test_conf.as_dict(display_sensitive=True)
self.assertEqual('cmd_result', cfg_dict['test']['key2'])
self.assertNotIn('key2_cmd', cfg_dict['test'])
# If we exclude _cmds then we should still see the commands to run, not
# their values
cfg_dict = test_conf.as_dict(include_cmds=False, display_sensitive=True)
self.assertNotIn('key4', cfg_dict['test'])
self.assertEqual('printf key4_result', cfg_dict['test']['key4_cmd'])
def test_getint(self):
"""Test AirflowConfigParser.getint"""
test_config = """
[invalid]
key1 = str
[valid]
key2 = 1
"""
test_conf = AirflowConfigParser(default_config=test_config)
with self.assertRaises(ValueError):
test_conf.getint('invalid', 'key1')
self.assertTrue(isinstance(test_conf.getint('valid', 'key2'), int))
self.assertEqual(1, test_conf.getint('valid', 'key2'))
def test_command_precedence(self):
TEST_CONFIG = '''[test]
key1 = hello
key2_cmd = printf cmd_result
key3 = airflow
key4_cmd = printf key4_result
'''
TEST_CONFIG_DEFAULT = '''[test]
key1 = awesome
key2 = airflow
[another]
key6 = value6
'''
test_conf = AirflowConfigParser(
default_config=parameterized_config(TEST_CONFIG_DEFAULT))
test_conf.read_string(TEST_CONFIG)
test_conf.as_command_stdout = test_conf.as_command_stdout | {
('test', 'key2'),
('test', 'key4'),
}
self.assertEqual('hello', test_conf.get('test', 'key1'))
self.assertEqual('cmd_result', test_conf.get('test', 'key2'))
self.assertEqual('airflow', test_conf.get('test', 'key3'))
self.assertEqual('key4_result', test_conf.get('test', 'key4'))
self.assertEqual('value6', test_conf.get('another', 'key6'))
self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb'))
self.assertEqual('value6',
test_conf.get('another', 'key6', fallback='fb'))
self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb'))
self.assertEqual(
True,
test_conf.getboolean('another', 'key8_boolean', fallback='True'))
self.assertEqual(
10, test_conf.getint('another', 'key8_int', fallback='10'))
self.assertEqual(
1.0, test_conf.getfloat('another', 'key8_float', fallback='1'))
self.assertTrue(test_conf.has_option('test', 'key1'))
self.assertTrue(test_conf.has_option('test', 'key2'))
self.assertTrue(test_conf.has_option('test', 'key3'))
self.assertTrue(test_conf.has_option('test', 'key4'))
self.assertFalse(test_conf.has_option('test', 'key5'))
self.assertTrue(test_conf.has_option('another', 'key6'))
cfg_dict = test_conf.as_dict(display_sensitive=True)
self.assertEqual('cmd_result', cfg_dict['test']['key2'])
self.assertNotIn('key2_cmd', cfg_dict['test'])
def test_getfloat(self):
"""Test AirflowConfigParser.getfloat"""
TEST_CONFIG = """
[invalid]
key1 = str
[valid]
key2 = 1.23
"""
test_conf = AirflowConfigParser(default_config=TEST_CONFIG)
with self.assertRaises(ValueError):
test_conf.getfloat('invalid', 'key1')
self.assertTrue(isinstance(test_conf.getfloat('valid', 'key2'), float))
self.assertEqual(1.23, test_conf.getfloat('valid', 'key2'))
def resolve_logs_folder() -> str:
"""
Returns LOGS folder specified in current Airflow config.
"""
config_file = get_airflow_config(AIRFLOW_HOME)
conf = AirflowConfigParser()
conf.read(config_file)
try:
logs = conf.get("logging", "base_log_folder")
except AirflowException:
try:
logs = conf.get("core", "base_log_folder")
except AirflowException:
logs = os.path.join(AIRFLOW_HOME, 'logs')
return logs
def test_kubernetes_environment_variables_section(self):
test_config = '''
[kubernetes_environment_variables]
key1 = hello
AIRFLOW_HOME = /root/airflow
'''
test_config_default = '''
[kubernetes_environment_variables]
'''
test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default))
test_conf.read_string(test_config)
assert OrderedDict([('key1', 'hello'), ('AIRFLOW_HOME', '/root/airflow')]) == test_conf.getsection(
'kubernetes_environment_variables'
)
def make_config():
test_conf = AirflowConfigParser(default_config='')
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
test_conf.deprecated_values = {
'core': {
'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'),
},
}
test_conf.read_dict({
'core': {
'executor': 'SequentialExecutor',
'task_runner': 'BashTaskRunner',
'sql_alchemy_conn': 'sqlite://',
},
})
return test_conf
def make_config():
test_conf = AirflowConfigParser(default_config='')
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
test_conf.deprecated_values = {
'core': {
'hostname_callable': (re.compile(r':'), r'.', '2.1'),
},
}
test_conf.read_dict({
'core': {
'executor': 'SequentialExecutor',
'sql_alchemy_conn': 'sqlite://',
'hostname_callable': 'socket:getfqdn',
},
})
return test_conf
def test_kubernetes_environment_variables_section(self):
TEST_CONFIG = '''
[kubernetes_environment_variables]
key1 = hello
AIRFLOW_HOME = /root/airflow
'''
TEST_CONFIG_DEFAULT = '''
[kubernetes_environment_variables]
'''
test_conf = AirflowConfigParser(
default_config=parameterized_config(TEST_CONFIG_DEFAULT))
test_conf.read_string(TEST_CONFIG)
self.assertEqual(
OrderedDict([('key1', 'hello'),
('AIRFLOW_HOME', '/root/airflow')]),
test_conf.getsection('kubernetes_environment_variables'))
def test_command_from_env(self):
test_cmdenv_config = '''[testcmdenv]
itsacommand = NOT OK
notacommand = OK
'''
test_cmdenv_conf = AirflowConfigParser()
test_cmdenv_conf.read_string(test_cmdenv_config)
test_cmdenv_conf.as_command_stdout.add(('testcmdenv', 'itsacommand'))
with unittest.mock.patch.dict('os.environ'):
# AIRFLOW__TESTCMDENV__ITSACOMMAND_CMD maps to ('testcmdenv', 'itsacommand') in
# as_command_stdout and therefore should return 'OK' from the environment variable's
# echo command, and must not return 'NOT OK' from the configuration
self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'itsacommand'), 'OK')
# AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD maps to no entry in as_command_stdout and therefore
# the option should return 'OK' from the configuration, and must not return 'NOT OK' from
# the environement variable's echo command
self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'notacommand'), 'OK')
def test_getfloat(self):
"""Test AirflowConfigParser.getfloat"""
test_config = """
[invalid]
key1 = str
[valid]
key2 = 1.23
"""
test_conf = AirflowConfigParser(default_config=test_config)
with self.assertRaisesRegex(
AirflowConfigException,
re.escape(
'Failed to convert value to float. Please check "key1" key in "invalid" section. '
'Current value: "str".')):
test_conf.getfloat('invalid', 'key1')
self.assertTrue(isinstance(test_conf.getfloat('valid', 'key2'), float))
self.assertEqual(1.23, test_conf.getfloat('valid', 'key2'))
def test_getfloat(self):
"""Test AirflowConfigParser.getfloat"""
test_config = """
[invalid]
key1 = str
[valid]
key2 = 1.23
"""
test_conf = AirflowConfigParser(default_config=test_config)
with pytest.raises(
AirflowConfigException,
match=re.escape(
'Failed to convert value to float. Please check "key1" key in "invalid" section. '
'Current value: "str".'),
):
test_conf.getfloat('invalid', 'key1')
assert isinstance(test_conf.getfloat('valid', 'key2'), float)
assert 1.23 == test_conf.getfloat('valid', 'key2')
def test_remove_option(self):
test_config = '''[test]
key1 = hello
key2 = airflow
'''
test_config_default = '''[test]
key1 = awesome
key2 = airflow
'''
test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default))
test_conf.read_string(test_config)
assert 'hello' == test_conf.get('test', 'key1')
test_conf.remove_option('test', 'key1', remove_default=False)
assert 'awesome' == test_conf.get('test', 'key1')
test_conf.remove_option('test', 'key2')
assert not test_conf.has_option('test', 'key2')
def test_remove_option(self):
TEST_CONFIG = '''[test]
key1 = hello
key2 = airflow
'''
TEST_CONFIG_DEFAULT = '''[test]
key1 = awesome
key2 = airflow
'''
test_conf = AirflowConfigParser(
default_config=parameterized_config(TEST_CONFIG_DEFAULT))
test_conf.read_string(TEST_CONFIG)
self.assertEqual('hello', test_conf.get('test', 'key1'))
test_conf.remove_option('test', 'key1', remove_default=False)
self.assertEqual('awesome', test_conf.get('test', 'key1'))
test_conf.remove_option('test', 'key2')
self.assertFalse(test_conf.has_option('test', 'key2'))
def test_config_from_secret_backend(self, mock_hvac):
"""Get Config Value from a Secret Backend"""
mock_client = mock.MagicMock()
mock_hvac.Client.return_value = mock_client
mock_client.secrets.kv.v2.read_secret_version.return_value = {
'request_id': '2d48a2ad-6bcb-e5b6-429d-da35fdf31f56',
'lease_id': '',
'renewable': False,
'lease_duration': 0,
'data': {
'data': {
'value': 'sqlite:////Users/airflow/airflow/airflow.db'
},
'metadata': {
'created_time': '2020-03-28T02:10:54.301784Z',
'deletion_time': '',
'destroyed': False,
'version': 1,
},
},
'wrap_info': None,
'warnings': None,
'auth': None,
}
test_config = '''[test]
sql_alchemy_conn_secret = sql_alchemy_conn
'''
test_config_default = '''[test]
sql_alchemy_conn = airflow
'''
test_conf = AirflowConfigParser(
default_config=parameterized_config(test_config_default))
test_conf.read_string(test_config)
test_conf.sensitive_config_values = test_conf.sensitive_config_values | {
('test', 'sql_alchemy_conn'),
}
self.assertEqual('sqlite:////Users/airflow/airflow/airflow.db',
test_conf.get('test', 'sql_alchemy_conn'))
def test_getsection(self):
TEST_CONFIG = '''
[test]
key1 = hello
'''
TEST_CONFIG_DEFAULT = '''
[test]
key1 = awesome
key2 = airflow
[another]
key3 = value3
'''
test_conf = AirflowConfigParser(
default_config=parameterized_config(TEST_CONFIG_DEFAULT))
test_conf.read_string(TEST_CONFIG)
self.assertEqual(OrderedDict([('key1', 'hello'), ('key2', 'airflow')]),
test_conf.getsection('test'))
self.assertEqual(OrderedDict([('key3', 'value3')]),
test_conf.getsection('another'))
def test_getsection(self):
test_config = '''
[test]
key1 = hello
'''
test_config_default = '''
[test]
key1 = awesome
key2 = airflow
[testsection]
key3 = value3
'''
test_conf = AirflowConfigParser(
default_config=parameterized_config(test_config_default))
test_conf.read_string(test_config)
self.assertEqual(OrderedDict([('key1', 'hello'), ('key2', 'airflow')]),
test_conf.getsection('test'))
self.assertEqual(
OrderedDict([('key3', 'value3'), ('testkey', 'testvalue'),
('testpercent', 'with%percent')]),
test_conf.getsection('testsection'))
def test_getboolean(self):
"""Test AirflowConfigParser.getboolean"""
test_config = """
[type_validation]
key1 = non_bool_value
[true]
key2 = t
key3 = true
key4 = 1
[false]
key5 = f
key6 = false
key7 = 0
[inline-comment]
key8 = true #123
"""
test_conf = AirflowConfigParser(default_config=test_config)
with pytest.raises(
AirflowConfigException,
match=re.escape(
'Failed to convert value to bool. Please check "key1" key in "type_validation" section. '
'Current value: "non_bool_value".'
),
):
test_conf.getboolean('type_validation', 'key1')
assert isinstance(test_conf.getboolean('true', 'key3'), bool)
assert test_conf.getboolean('true', 'key2') is True
assert test_conf.getboolean('true', 'key3') is True
assert test_conf.getboolean('true', 'key4') is True
assert test_conf.getboolean('false', 'key5') is False
assert test_conf.getboolean('false', 'key6') is False
assert test_conf.getboolean('false', 'key7') is False
assert test_conf.getboolean('inline-comment', 'key8') is True
