def test_invalid_env_var_raises_error(self, monkeypatch):
monkeypatch.setenv("PREFECT_TEST__X__Y__KEYS__Z", "TEST")
with tempfile.NamedTemporaryFile() as test_config:
with pytest.raises(ValueError):
configuration.load_configuration(test_config.name,
env_var_prefix="PREFECT_TEST")
def test_invalid_env_var_raises_error(self, monkeypatch):
monkeypatch.setenv("PREFECT_TEST__X__Y__KEYS__Z", "TEST")
with tempfile.TemporaryDirectory() as test_config_dir:
test_config_loc = os.path.join(test_config_dir, "test_config.toml")
with open(test_config_loc, "wb") as test_config:
test_config.write(b"")
with pytest.raises(ValueError):
configuration.load_configuration(test_config_loc,
env_var_prefix="PREFECT_TEST")
def test_non_lowercase_keys_raise_error(self):
with tempfile.NamedTemporaryFile() as test_config:
test_config.write(b"""
[KEY]
x = 1
""")
test_config.seek(0)
with pytest.raises(ValueError):
configuration.load_configuration(test_config.name)
def test_invalid_keys_raise_error(self):
with tempfile.NamedTemporaryFile() as test_config:
test_config.write(b"""
[outer]
x = 1
[outer.keys]
a = "b"
""")
test_config.seek(0)
with pytest.raises(ValueError):
configuration.load_configuration(test_config.name)
def test_invalid_keys_raise_error(self):
with tempfile.TemporaryDirectory() as test_config_dir:
test_config_loc = os.path.join(test_config_dir, "test_config.toml")
with open(test_config_loc, "wb") as test_config:
test_config.write(b"""
[outer]
x = 1
[outer.keys]
a = "b"
""")
with pytest.raises(ValueError):
configuration.load_configuration(test_config_loc)
def test_load_user_config(self, test_config_file_path):
with tempfile.TemporaryDirectory() as user_config_dir:
user_config_loc = os.path.join(user_config_dir, "test_config.toml")
with open(user_config_loc, "wb") as user_config:
user_config.write(
b"""
[general]
x = 2
[user]
foo = "bar"
"""
)
config = configuration.load_configuration(
path=test_config_file_path, user_config_path=user_config_loc
)
# check that user values are loaded
assert config.general.x == 2
assert config.user.foo == "bar"
# check that default values are preserved
assert config.general.y == "hi"
# check that interpolation takes place after user config is loaded
assert config.general.nested.x == 2
def test_env_vars_for_secrets_alone_are_not_lower_cased(self, monkeypatch):
monkeypatch.setenv("PREFECT_TEST__CONTEXT__SECRETS__mY_spECIal_kEY",
"42")
monkeypatch.setenv("PREFECT_TEST__CONTEXT__strange_VAlUE", "false")
monkeypatch.setenv("PREFECT_TEST__CONTEXT__FLOW_RUN_ID", "12345")
monkeypatch.setenv("PREFECT_TEST__CONTEXT__FLOW_ID", "56789")
with tempfile.TemporaryDirectory() as test_config_dir:
test_config_loc = os.path.join(test_config_dir, "test_config.toml")
with open(test_config_loc, "wb") as test_config:
test_config.write(b"""
[context]
spECIAL_TOP_key = "foo"
[context.secrets]
KeY = 1
""")
config = configuration.load_configuration(
test_config_loc, env_var_prefix="PREFECT_TEST")
assert "secrets" in config.context
assert config.context.secrets.mY_spECIal_kEY == 42
assert config.context.secrets.KeY == 1
assert config.context.spECIAL_TOP_key == "foo"
assert config.context.flow_run_id == 12345
assert config.context.flow_id == 56789
assert config.context.strange_value is False
def test_load_user_config(self, test_config_file_path):
with tempfile.NamedTemporaryFile() as user_config:
user_config.write(b"""
[general]
x = 2
[user]
foo = "bar"
""")
user_config.seek(0)
test_config = configuration.load_configuration(
test_config_file_path)
config = configuration.load_configuration(
user_config.name, merge_into_config=test_config)
assert config.general.x == 2
assert config.user.foo == "bar"
def test_merge_configurations(test_config_file_path):
default_config = configuration.config
assert default_config.logging.format != "log-format"
config = configuration.load_configuration(
config_path=test_config_file_path, merge_into_config=default_config)
assert config.logging.format == "log-format"
assert config.interpolation.value == 1
def test_mixed_case_keys_are_ok(self):
with tempfile.NamedTemporaryFile() as test_config:
test_config.write(b"""
[SeCtIoN]
KeY = 1
""")
test_config.seek(0)
config = configuration.load_configuration(test_config.name)
assert "KeY" in config.SeCtIoN
assert config.SeCtIoN.KeY == 1
def test_non_lowercase_keys_are_ok_in_context(self):
with tempfile.NamedTemporaryFile() as test_config:
test_config.write(b"""
[context]
KeY = 1
""")
test_config.seek(0)
config = configuration.load_configuration(test_config.name)
assert "KeY" in config.context
assert config.context.KeY == 1
def test_mixed_case_keys_are_ok(self):
with tempfile.TemporaryDirectory() as test_config_dir:
test_config_loc = os.path.join(test_config_dir, "test_config.toml")
with open(test_config_loc, "wb") as test_config:
test_config.write(b"""
[SeCtIoN]
KeY = 1
""")
config = configuration.load_configuration(test_config_loc)
assert "KeY" in config.SeCtIoN
assert config.SeCtIoN.KeY == 1
def test_non_lowercase_keys_are_ok_in_context_subsections(self):
with tempfile.NamedTemporaryFile() as test_config:
test_config.write(b"""
[context.secrets]
API_CREDS = 4
""")
test_config.seek(0)
config = configuration.load_configuration(test_config.name)
assert "secrets" in config.context
assert "API_CREDS" in config.context.secrets
assert config.context.secrets["API_CREDS"] == 4
def test_to_environment_variables_roundtrip(config, monkeypatch,
test_config_file_path):
keys = [".".join(k) for k in dict_to_flatdict(config)]
# Note prefix is different to avoid colliding with the `config` fixture env
env = to_environment_variables(config,
include=keys,
prefix="PREFECT_TEST_ROUND")
for k, v in env.items():
monkeypatch.setenv(k, v)
new_config = configuration.load_configuration(
test_config_file_path, env_var_prefix="PREFECT_TEST_ROUND")
assert new_config == config
def config(test_config_file_path, monkeypatch):
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEW_KEY", "TEST")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__TWICE__NESTED__NEW_KEY",
"TEST")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__TRUE", "true")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__FALSE", "false")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__INT", "10")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEGATIVE_INT", "-10")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__FLOAT", "7.5")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEGATIVE_FLOAT", "-7.5")
monkeypatch.setenv("PATH", "1/2/3")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__ESCAPED_CHARACTERS",
r"line 1\nline 2\rand 3\tand 4")
yield configuration.load_configuration(test_config_file_path,
env_var_prefix="PREFECT_TEST")
def test_env_vars_are_interpolated_as_lower_case(self, monkeypatch):
monkeypatch.setenv("PREFECT_TEST__SECTION__KEY", "2")
with tempfile.NamedTemporaryFile() as test_config:
test_config.write(b"""
[SeCtIoN]
KeY = 1
""")
test_config.seek(0)
config = configuration.load_configuration(
test_config.name, env_var_prefix="PREFECT_TEST")
assert "KeY" in config.SeCtIoN
assert config.SeCtIoN.KeY == 1
assert config.section.key == 2
def test_env_vars_are_interpolated_as_lower_case(self, monkeypatch):
monkeypatch.setenv("PREFECT_TEST__SECTION__KEY", "2")
with tempfile.TemporaryDirectory() as test_config_dir:
test_config_loc = os.path.join(test_config_dir, "test_config.toml")
with open(test_config_loc, "wb") as test_config:
test_config.write(b"""
[SeCtIoN]
KeY = 1
""")
config = configuration.load_configuration(
test_config_loc, env_var_prefix="PREFECT_TEST")
assert "KeY" in config.SeCtIoN
assert config.SeCtIoN.KeY == 1
assert config.section.key == 2
def config(test_config_file_path, monkeypatch):
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEW_KEY", "TEST")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__TWICE__NESTED__NEW_KEY",
"TEST")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__TRUE", "true")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__FALSE", "false")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__INT", "10")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEGATIVE_INT", "-10")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__FLOAT", "7.5")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEGATIVE_FLOAT", "-7.5")
monkeypatch.setenv(
"PREFECT_TEST__CONTEXT__SECRETS__AWS_CREDENTIALS",
'{"ACCESS_KEY": "abcdef", "SECRET_ACCESS_KEY": "ghijklmn"}',
)
monkeypatch.setenv("PATH", "1/2/3")
monkeypatch.setenv("PREFECT_TEST__ENV_VARS__ESCAPED_CHARACTERS",
r"line 1\nline 2\rand 3\tand 4")
yield configuration.load_configuration(test_config_file_path,
env_var_prefix="PREFECT_TEST")
def test_load_user_config(self, test_config_file_path):
with tempfile.NamedTemporaryFile() as user_config:
user_config.write(b"""
[general]
x = 2
[user]
foo = "bar"
""")
user_config.seek(0)
config = configuration.load_configuration(
path=test_config_file_path, user_config_path=user_config.name)
# check that user values are loaded
assert config.general.x == 2
assert config.user.foo == "bar"
# check that default values are preserved
assert config.general.y == "hi"
# check that interpolation takes place after user config is loaded
assert config.general.nested.x == 2
# Licensed under the Prefect Community License, available at
# https://www.prefect.io/legal/prefect-community-license
import os
import prefect_server
from prefect.configuration import load_configuration
DEFAULT_CONFIG = os.path.join(os.path.dirname(prefect_server.__file__),
"config.toml")
USER_CONFIG = os.getenv("PREFECT_SERVER__USER_CONFIG_PATH",
"~/.prefect_server/config.toml")
ENV_VAR_PREFIX = "PREFECT_SERVER"
config = load_configuration(path=DEFAULT_CONFIG,
user_config_path=USER_CONFIG,
env_var_prefix=ENV_VAR_PREFIX)
