def test_resolve_xcom_class_fallback_to_basexcom_no_config(self):
init = conf.get("core", "xcom_backend")
conf.remove_option("core", "xcom_backend")
cls = resolve_xcom_backend()
assert issubclass(cls, BaseXCom)
assert cls().serialize_value([1]) == b"[1]"
conf.set("core", "xcom_backend", init)
def conf_vars(overrides):
original = {}
original_env_vars = {}
for (section, key), value in overrides.items():
env = conf._env_var_name(section, key)
if env in os.environ:
original_env_vars[env] = os.environ.pop(env)
if conf.has_option(section, key):
original[(section, key)] = conf.get(section, key)
else:
original[(section, key)] = None
if value is not None:
conf.set(section, key, value)
else:
conf.remove_option(section, key)
settings.configure_vars()
try:
yield
finally:
for (section, key), value in original.items():
if value is not None:
conf.set(section, key, value)
else:
conf.remove_option(section, key)
for env, value in original_env_vars.items():
os.environ[env] = value
settings.configure_vars()
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
# test env vars
self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
# test defaults
conf.remove_option('core', 'load_examples')
cfg_dict = conf.as_dict()
self.assertEqual(cfg_dict['core']['load_examples'], 'True')
# test display_source
cfg_dict = conf.as_dict(display_source=True)
self.assertEqual(cfg_dict['core']['unit_test_mode'][1], 'airflow.cfg')
self.assertEqual(cfg_dict['core']['load_examples'][1], 'default')
self.assertEqual(cfg_dict['testsection']['testkey'],
('< hidden >', 'env var'))
# test display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True)
self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
# test display_source and display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
self.assertEqual(cfg_dict['testsection']['testkey'],
('testvalue', 'env var'))
def conf_vars(overrides):
original = {}
original_env_vars = {}
reconfigure_vars = False
for (section, key), value in overrides.items():
env = conf._env_var_name(section, key)
if env in os.environ:
original_env_vars[env] = os.environ.pop(env)
if conf.has_option(section, key):
original[(section, key)] = conf.get(section, key)
else:
original[(section, key)] = None
if value is not None:
conf.set(section, key, value)
else:
conf.remove_option(section, key)
if section == 'core' and key.lower().endswith('_folder'):
reconfigure_vars = True
if reconfigure_vars:
settings.configure_vars()
yield
for (section, key), value in original.items():
if value is not None:
conf.set(section, key, value)
else:
conf.remove_option(section, key)
for env, value in original_env_vars.items():
os.environ[env] = value
if reconfigure_vars:
settings.configure_vars()
def test_conf_as_dict(self):
cfg_dict = conf.as_dict()
# test that configs are picked up
self.assertEqual(cfg_dict["core"]["unit_test_mode"], "True")
# test env vars
self.assertEqual(cfg_dict["testsection"]["testkey"], "< hidden >")
# test defaults
conf.remove_option("core", "load_examples")
cfg_dict = conf.as_dict()
self.assertEqual(cfg_dict["core"]["load_examples"], "True")
# test display_source
cfg_dict = conf.as_dict(display_source=True)
self.assertEqual(cfg_dict["core"]["unit_test_mode"][1], "airflow.cfg")
self.assertEqual(cfg_dict["core"]["load_examples"][1], "default")
self.assertEqual(cfg_dict["testsection"]["testkey"], ("< hidden >", "env var"))
# test display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True)
self.assertEqual(cfg_dict["testsection"]["testkey"], "testvalue")
# test display_source and display_sensitive
cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
self.assertEqual(cfg_dict["testsection"]["testkey"], ("testvalue", "env var"))
def test_deprecated_options_cmd(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options['celery'] = {'result_backend': 'celery_result_backend'}
conf.as_command_stdout.add(('celery', 'celery_result_backend'))
conf.remove_option('celery', 'result_backend')
conf.set('celery', 'celery_result_backend_cmd', '/bin/echo 99')
with self.assertWarns(DeprecationWarning):
self.assertEquals(conf.getint('celery', 'result_backend'), 99)
def test_deprecated_options_cmd(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options['celery'] = {'result_backend': 'celery_result_backend'}
conf.as_command_stdout.add(('celery', 'celery_result_backend'))
conf.remove_option('celery', 'result_backend')
conf.set('celery', 'celery_result_backend_cmd', '/bin/echo 99')
with self.assertWarns(DeprecationWarning):
self.assertEquals(conf.getint('celery', 'result_backend'), 99)
def test_deprecated_options_cmd(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options[('celery', "result_backend")] = 'celery', 'celery_result_backend', '2.0.0'
conf.sensitive_config_values.add(('celery', 'celery_result_backend'))
conf.remove_option('celery', 'result_backend')
with conf_vars({('celery', 'celery_result_backend_cmd'): '/bin/echo 99'}):
with pytest.warns(DeprecationWarning):
tmp = None
if 'AIRFLOW__CELERY__RESULT_BACKEND' in os.environ:
tmp = os.environ.pop('AIRFLOW__CELERY__RESULT_BACKEND')
assert conf.getint('celery', 'result_backend') == 99
if tmp:
os.environ['AIRFLOW__CELERY__RESULT_BACKEND'] = tmp
def test_deprecated_options_cmd(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options[('celery', "result_backend")] = ('celery', 'celery_result_backend')
conf.as_command_stdout.add(('celery', 'celery_result_backend'))
conf.remove_option('celery', 'result_backend')
conf.set('celery', 'celery_result_backend_cmd', '/bin/echo 99')
with self.assertWarns(DeprecationWarning):
tmp = None
if 'AIRFLOW__CELERY__RESULT_BACKEND' in os.environ:
tmp = os.environ.pop('AIRFLOW__CELERY__RESULT_BACKEND')
self.assertEqual(conf.getint('celery', 'result_backend'), 99)
if tmp:
os.environ['AIRFLOW__CELERY__RESULT_BACKEND'] = tmp
def test_deprecated_options_cmd(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options['celery'] = {'result_backend': 'celery_result_backend'}
conf.as_command_stdout.add(('celery', 'celery_result_backend'))
conf.remove_option('celery', 'result_backend')
conf.set('celery', 'celery_result_backend_cmd', '/bin/echo 99')
with self.assertWarns(DeprecationWarning):
tmp = None
if 'AIRFLOW__CELERY__RESULT_BACKEND' in os.environ:
tmp = os.environ.pop('AIRFLOW__CELERY__RESULT_BACKEND')
self.assertEqual(conf.getint('celery', 'result_backend'), 99)
if tmp:
os.environ['AIRFLOW__CELERY__RESULT_BACKEND'] = tmp
def test_deprecated_options(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options = {
('celery', 'worker_concurrency'): ('celery', 'celeryd_concurrency', '2.0.0'),
}
# Remove it so we are sure we use the right setting
conf.remove_option('celery', 'worker_concurrency')
with pytest.warns(DeprecationWarning):
with mock.patch.dict('os.environ', AIRFLOW__CELERY__CELERYD_CONCURRENCY="99"):
assert conf.getint('celery', 'worker_concurrency') == 99
with pytest.warns(DeprecationWarning), conf_vars({('celery', 'celeryd_concurrency'): '99'}):
assert conf.getint('celery', 'worker_concurrency') == 99
def conf_vars(overrides):
original = {}
for (section, key), value in overrides.items():
if conf.has_option(section, key):
original[(section, key)] = conf.get(section, key)
else:
original[(section, key)] = None
if value is not None:
conf.set(section, key, value)
else:
conf.remove_option(section, key)
yield
for (section, key), value in original.items():
if value is not None:
conf.set(section, key, value)
else:
conf.remove_option(section, key)
def test_deprecated_options_with_new_section(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options = {
('logging', 'logging_level'): ('core', 'logging_level', '2.0.0'),
}
# Remove it so we are sure we use the right setting
conf.remove_option('core', 'logging_level')
conf.remove_option('logging', 'logging_level')
with pytest.warns(DeprecationWarning):
with mock.patch.dict('os.environ', AIRFLOW__CORE__LOGGING_LEVEL="VALUE"):
assert conf.get('logging', 'logging_level') == "VALUE"
with pytest.warns(DeprecationWarning), conf_vars({('core', 'logging_level'): 'VALUE'}):
assert conf.get('logging', 'logging_level') == "VALUE"
def test_deprecated_options(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options = {
('celery', 'worker_concurrency'): ('celery', 'celeryd_concurrency'),
}
# Remove it so we are sure we use the right setting
conf.remove_option('celery', 'worker_concurrency')
with self.assertWarns(DeprecationWarning):
with mock.patch.dict('os.environ', AIRFLOW__CELERY__CELERYD_CONCURRENCY="99"):
self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
with self.assertWarns(DeprecationWarning):
conf.set('celery', 'celeryd_concurrency', '99')
self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
conf.remove_option('celery', 'celeryd_concurrency')
def test_deprecated_options(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options['celery'] = {
'worker_concurrency': 'celeryd_concurrency',
}
# Remove it so we are sure we use the right setting
conf.remove_option('celery', 'worker_concurrency')
with self.assertWarns(DeprecationWarning):
os.environ['AIRFLOW__CELERY__CELERYD_CONCURRENCY'] = '99'
self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
os.environ.pop('AIRFLOW__CELERY__CELERYD_CONCURRENCY')
with self.assertWarns(DeprecationWarning):
conf.set('celery', 'celeryd_concurrency', '99')
self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
conf.remove_option('celery', 'celeryd_concurrency')
def test_deprecated_options(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options['celery'] = {
'worker_concurrency': 'celeryd_concurrency',
}
# Remove it so we are sure we use the right setting
conf.remove_option('celery', 'worker_concurrency')
with self.assertWarns(DeprecationWarning):
os.environ['AIRFLOW__CELERY__CELERYD_CONCURRENCY'] = '99'
self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
os.environ.pop('AIRFLOW__CELERY__CELERYD_CONCURRENCY')
with self.assertWarns(DeprecationWarning):
conf.set('celery', 'celeryd_concurrency', '99')
self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
conf.remove_option('celery', 'celeryd_concurrency')
def setUp(self):
conf.remove_option('email', 'EMAIL_BACKEND')
conf.remove_option('email', 'EMAIL_CONN_ID')
def test_should_stop_app_when_removed_options_are_provided(self):
with self.assertRaises(SystemExit) as e:
conf.remove_option('webserver', 'session_lifetime_minutes')
application.cached_app(testing=True)
self.assertEqual(e.exception.code, 4)