def test_command_config(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6'))
def test_command_config(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6'))
def resolve_logs_folder() -> str: """ Returns LOGS folder specified in current Airflow config. """ config_file = get_airflow_config(AIRFLOW_HOME) conf = AirflowConfigParser() conf.read(config_file) try: logs = conf.get("logging", "base_log_folder") except AirflowException: try: logs = conf.get("core", "base_log_folder") except AirflowException: logs = os.path.join(AIRFLOW_HOME, 'logs') return logs
def test_config_from_secret_backend(self, mock_hvac): """Get Config Value from a Secret Backend""" mock_client = mock.MagicMock() mock_hvac.Client.return_value = mock_client mock_client.secrets.kv.v2.read_secret_version.return_value = { 'request_id': '2d48a2ad-6bcb-e5b6-429d-da35fdf31f56', 'lease_id': '', 'renewable': False, 'lease_duration': 0, 'data': {'data': {'value': 'sqlite:////Users/airflow/airflow/airflow.db'}, 'metadata': {'created_time': '2020-03-28T02:10:54.301784Z', 'deletion_time': '', 'destroyed': False, 'version': 1}}, 'wrap_info': None, 'warnings': None, 'auth': None } test_config = '''[test] sql_alchemy_conn_secret = sql_alchemy_conn ''' test_config_default = '''[test] sql_alchemy_conn = airflow ''' test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) test_conf.sensitive_config_values = test_conf.sensitive_config_values | { ('test', 'sql_alchemy_conn'), } self.assertEqual( 'sqlite:////Users/airflow/airflow/airflow.db', test_conf.get('test', 'sql_alchemy_conn'))
def test_command_precedence(self): test_config = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' test_config_default = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) test_conf.sensitive_config_values = test_conf.sensitive_config_values | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb')) self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb')) self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb')) self.assertEqual( True, test_conf.getboolean('another', 'key8_boolean', fallback='True')) self.assertEqual( 10, test_conf.getint('another', 'key8_int', fallback='10')) self.assertEqual( 1.0, test_conf.getfloat('another', 'key8_float', fallback='1')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6')) cfg_dict = test_conf.as_dict(display_sensitive=True) self.assertEqual('cmd_result', cfg_dict['test']['key2']) self.assertNotIn('key2_cmd', cfg_dict['test']) # If we exclude _cmds then we should still see the commands to run, not # their values cfg_dict = test_conf.as_dict(include_cmds=False, display_sensitive=True) self.assertNotIn('key4', cfg_dict['test']) self.assertEqual('printf key4_result', cfg_dict['test']['key4_cmd'])
def test_command_from_env(self): test_cmdenv_config = '''[testcmdenv] itsacommand = NOT OK notacommand = OK ''' test_cmdenv_conf = AirflowConfigParser() test_cmdenv_conf.read_string(test_cmdenv_config) test_cmdenv_conf.as_command_stdout.add(('testcmdenv', 'itsacommand')) with unittest.mock.patch.dict('os.environ'): # AIRFLOW__TESTCMDENV__ITSACOMMAND_CMD maps to ('testcmdenv', 'itsacommand') in # as_command_stdout and therefore should return 'OK' from the environment variable's # echo command, and must not return 'NOT OK' from the configuration self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'itsacommand'), 'OK') # AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD maps to no entry in as_command_stdout and therefore # the option should return 'OK' from the configuration, and must not return 'NOT OK' from # the environement variable's echo command self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'notacommand'), 'OK')
def test_remove_option(self): test_config = '''[test] key1 = hello key2 = airflow ''' test_config_default = '''[test] key1 = awesome key2 = airflow ''' test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default)) test_conf.read_string(test_config) assert 'hello' == test_conf.get('test', 'key1') test_conf.remove_option('test', 'key1', remove_default=False) assert 'awesome' == test_conf.get('test', 'key1') test_conf.remove_option('test', 'key2') assert not test_conf.has_option('test', 'key2')
def test_remove_option(self): TEST_CONFIG = '''[test] key1 = hello key2 = airflow ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) self.assertEqual('hello', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key1', remove_default=False) self.assertEqual('awesome', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key2') self.assertFalse(test_conf.has_option('test', 'key2'))
def test_remove_option(self): TEST_CONFIG = '''[test] key1 = hello key2 = airflow ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) self.assertEqual('hello', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key1', remove_default=False) self.assertEqual('awesome', test_conf.get('test', 'key1')) test_conf.remove_option('test', 'key2') self.assertFalse(test_conf.has_option('test', 'key2'))
def resolve_dags_folder() -> str: """ Returns DAG folder specified in current Airflow config. """ config_file = get_airflow_config(AIRFLOW_HOME) conf = AirflowConfigParser() conf.read(config_file) try: dags = conf.get("core", "dags_folder") except AirflowException: dags = os.path.join(AIRFLOW_HOME, 'dags') return dags
def test_command_precedence(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb')) self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb')) self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb')) self.assertEqual( True, test_conf.getboolean('another', 'key8_boolean', fallback='True')) self.assertEqual( 10, test_conf.getint('another', 'key8_int', fallback='10')) self.assertEqual( 1.0, test_conf.getfloat('another', 'key8_float', fallback='1')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6')) cfg_dict = test_conf.as_dict(display_sensitive=True) self.assertEqual('cmd_result', cfg_dict['test']['key2']) self.assertNotIn('key2_cmd', cfg_dict['test'])
def test_command_precedence(self): TEST_CONFIG = '''[test] key1 = hello key2_cmd = printf cmd_result key3 = airflow key4_cmd = printf key4_result ''' TEST_CONFIG_DEFAULT = '''[test] key1 = awesome key2 = airflow [another] key6 = value6 ''' test_conf = AirflowConfigParser( default_config=parameterized_config(TEST_CONFIG_DEFAULT)) test_conf.read_string(TEST_CONFIG) test_conf.as_command_stdout = test_conf.as_command_stdout | { ('test', 'key2'), ('test', 'key4'), } self.assertEqual('hello', test_conf.get('test', 'key1')) self.assertEqual('cmd_result', test_conf.get('test', 'key2')) self.assertEqual('airflow', test_conf.get('test', 'key3')) self.assertEqual('key4_result', test_conf.get('test', 'key4')) self.assertEqual('value6', test_conf.get('another', 'key6')) self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb')) self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb')) self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb')) self.assertEqual(True, test_conf.getboolean('another', 'key8_boolean', fallback='True')) self.assertEqual(10, test_conf.getint('another', 'key8_int', fallback='10')) self.assertEqual(1.0, test_conf.getfloat('another', 'key8_float', fallback='1')) self.assertTrue(test_conf.has_option('test', 'key1')) self.assertTrue(test_conf.has_option('test', 'key2')) self.assertTrue(test_conf.has_option('test', 'key3')) self.assertTrue(test_conf.has_option('test', 'key4')) self.assertFalse(test_conf.has_option('test', 'key5')) self.assertTrue(test_conf.has_option('another', 'key6')) cfg_dict = test_conf.as_dict(display_sensitive=True) self.assertEqual('cmd_result', cfg_dict['test']['key2']) self.assertNotIn('key2_cmd', cfg_dict['test'])