def test_getsection(self): test_config = ''' [test] key1 = hello ''' test_config_default = ''' [test] key1 = awesome key2 = airflow [testsection] key3 = value3 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) self.assertEqual( OrderedDict([('key1', 'hello'), ('key2', 'airflow')]), test_conf.getsection('test') ) self.assertEqual( OrderedDict([ ('key3', 'value3'), ('testkey', 'testvalue'), ('testpercent', 'with%percent')]), test_conf.getsection('testsection') )
def test_command_config(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6'))
def test_getsection(self): test_config = ''' [test] key1 = hello [new_section] key = value ''' test_config_default = ''' [test] key1 = awesome key2 = airflow [testsection] key3 = value3 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) assert OrderedDict([('key1', 'hello'), ('key2', 'airflow') ]) == test_conf.getsection('test') assert OrderedDict([('key3', 'value3'), ('testkey', 'testvalue'), ('testpercent', 'with%percent') ]) == test_conf.getsection('testsection') assert OrderedDict([('key', 'value') ]) == test_conf.getsection('new_section') assert test_conf.getsection('non_existent_section') is None
def test_getsection(self): TEST_CONFIG = ''' [test] key1 = hello ''' TEST_CONFIG_DEFAULT = ''' [test] key1 = awesome key2 = airflow [another] key3 = value3 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) self.assertEqual( OrderedDict([('key1', 'hello'), ('key2', 'airflow')]), test_conf.getsection('test') ) self.assertEqual( OrderedDict([('key3', 'value3')]), test_conf.getsection('another') )
def test_getsection(self): TEST_CONFIG = ''' [test] key1 = hello ''' TEST_CONFIG_DEFAULT = ''' [test] key1 = awesome key2 = airflow [another] key3 = value3 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) self.assertEqual( OrderedDict([('key1', 'hello'), ('key2', 'airflow')]), test_conf.getsection('test') ) self.assertEqual( OrderedDict([('key3', 'value3')]), test_conf.getsection('another') )
def test_command_config(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6'))
def test_config_from_secret_backend(self, mock_hvac): """Get Config Value from a Secret Backend""" mock_client = mock.MagicMock() mock_hvac.Client.return_value = mock_client mock_client.secrets.kv.v2.read_secret_version.return_value = { 'request_id': '2d48a2ad-6bcb-e5b6-429d-da35fdf31f56', 'lease_id': '', 'renewable': False, 'lease_duration': 0, 'data': {'data': {'value': 'sqlite:////Users/airflow/airflow/airflow.db'}, 'metadata': {'created_time': '2020-03-28T02:10:54.301784Z', 'deletion_time': '', 'destroyed': False, 'version': 1}}, 'wrap_info': None, 'warnings': None, 'auth': None } test_config = '''[test] sql_alchemy_conn_secret = sql_alchemy_conn ''' test_config_default = '''[test] sql_alchemy_conn = airflow ''' test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) test_conf.sensitive_config_values = test_conf.sensitive_config_values | { ('test', 'sql_alchemy_conn'), } self.assertEqual( 'sqlite:////Users/airflow/airflow/airflow.db', test_conf.get('test', 'sql_alchemy_conn'))
def patch_config(config): parser = AirflowConfigParser() parser.read_string(textwrap.dedent(config)) with patch("airflow.configuration.get", new=parser.get): with patch("airflow.configuration.has_option", new=parser.has_option): yield
def test_command_precedence(self): test_config = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' test_config_default = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) test_conf.sensitive_config_values = test_conf.sensitive_config_values | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb')) self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb')) self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb')) self.assertEqual( True, test_conf.getboolean('another', 'key8_boolean', fallback='True')) self.assertEqual( 10, test_conf.getint('another', 'key8_int', fallback='10')) self.assertEqual( 1.0, test_conf.getfloat('another', 'key8_float', fallback='1')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6')) cfg_dict = test_conf.as_dict(display_sensitive=True) self.assertEqual('cmd_result', cfg_dict['test']['key2']) self.assertNotIn('key2_cmd', cfg_dict['test']) # If we exclude _cmds then we should still see the commands to run, not # their values cfg_dict = test_conf.as_dict(include_cmds=False, display_sensitive=True) self.assertNotIn('key4', cfg_dict['test']) self.assertEqual('printf key4_result', cfg_dict['test']['key4_cmd'])
def test_has_option(self): test_config = '''[test] key1 = value1 ''' test_conf = AirflowConfigParser() test_conf.read_string(test_config) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertFalse(test_conf.has_option('test', 'key_not_exists')) self.assertFalse(test_conf.has_option('section_not_exists', 'key1'))
def test_command_precedence(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb')) self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb')) self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb')) self.assertEqual( True, test_conf.getboolean('another', 'key8_boolean', fallback='True')) self.assertEqual( 10, test_conf.getint('another', 'key8_int', fallback='10')) self.assertEqual( 1.0, test_conf.getfloat('another', 'key8_float', fallback='1')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6')) cfg_dict = test_conf.as_dict(display_sensitive=True) self.assertEqual('cmd_result', cfg_dict['test']['key2']) self.assertNotIn('key2_cmd', cfg_dict['test'])
def test_kubernetes_environment_variables_section(self): test_config = ''' [kubernetes_environment_variables] key1 = hello AIRFLOW_HOME = /root/airflow ''' test_config_default = ''' [kubernetes_environment_variables] ''' test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) assert OrderedDict([('key1', 'hello'), ('AIRFLOW_HOME', '/root/airflow')]) == test_conf.getsection( 'kubernetes_environment_variables' )
def test_command_from_env(self): test_cmdenv_config = '''[testcmdenv] itsacommand = NOT OK notacommand = OK ''' test_cmdenv_conf = AirflowConfigParser() test_cmdenv_conf.read_string(test_cmdenv_config) test_cmdenv_conf.as_command_stdout.add(('testcmdenv', 'itsacommand')) with unittest.mock.patch.dict('os.environ'): # AIRFLOW__TESTCMDENV__ITSACOMMAND_CMD maps to ('testcmdenv', 'itsacommand') in # as_command_stdout and therefore should return 'OK' from the environment variable's # echo command, and must not return 'NOT OK' from the configuration self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'itsacommand'), 'OK') # AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD maps to no entry in as_command_stdout and therefore # the option should return 'OK' from the configuration, and must not return 'NOT OK' from # the environement variable's echo command self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'notacommand'), 'OK')
def test_kubernetes_environment_variables_section(self): TEST_CONFIG = ''' [kubernetes_environment_variables] key1 = hello AIRFLOW_HOME = /root/airflow ''' TEST_CONFIG_DEFAULT = ''' [kubernetes_environment_variables] ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) self.assertEqual( OrderedDict([('key1', 'hello'), ('AIRFLOW_HOME', '/root/airflow')]), test_conf.getsection('kubernetes_environment_variables'))
def test_command_precedence(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb')) self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb')) self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb')) self.assertEqual(True, test_conf.getboolean('another', 'key8_boolean', fallback='True')) self.assertEqual(10, test_conf.getint('another', 'key8_int', fallback='10')) self.assertEqual(1.0, test_conf.getfloat('another', 'key8_float', fallback='1')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6')) cfg_dict = test_conf.as_dict(display_sensitive=True) self.assertEqual('cmd_result', cfg_dict['test']['key2']) self.assertNotIn('key2_cmd', cfg_dict['test'])
def test_remove_option(self): test_config = '''[test] key1 = hello key2 = airflow ''' test_config_default = '''[test] key1 = awesome key2 = airflow ''' test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) assert 'hello' == test_conf.get('test', 'key1') test_conf.remove_option('test', 'key1', remove_default=False) assert 'awesome' == test_conf.get('test', 'key1') test_conf.remove_option('test', 'key2') assert not test_conf.has_option('test', 'key2')
def test_remove_option(self): TEST_CONFIG = '''[test] key1 = hello key2 = airflow ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) self.assertEqual('hello', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key1', remove_default=False) self.assertEqual('awesome', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key2') self.assertFalse(test_conf.has_option('test', 'key2'))
def test_remove_option(self): TEST_CONFIG = '''[test] key1 = hello key2 = airflow ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) self.assertEqual('hello', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key1', remove_default=False) self.assertEqual('awesome', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key2') self.assertFalse(test_conf.has_option('test', 'key2'))