class Platform(object):
SECTION = "platform"
URL = ConfigEntry(
LegacyConfigEntry(SECTION, "url"), YamlConfigEntry("admin.endpoint"), lambda x: x.replace("dns:///", "")
)
INSECURE = ConfigEntry(LegacyConfigEntry(SECTION, "insecure", bool), YamlConfigEntry("admin.insecure", bool))
INSECURE_SKIP_VERIFY = ConfigEntry(
LegacyConfigEntry(SECTION, "insecure_skip_verify", bool), YamlConfigEntry("admin.insecureSkipVerify", bool)
)
class StatsD(object):
SECTION = "secrets"
# StatsD Config flags should ideally be controlled at the platform level and not through flytekit's config file.
# They are meant to allow administrators to control certain behavior according to how the system is configured.
HOST = ConfigEntry(LegacyConfigEntry(SECTION, "host"))
PORT = ConfigEntry(LegacyConfigEntry(SECTION, "port", int))
DISABLED = ConfigEntry(LegacyConfigEntry(SECTION, "disabled", bool))
DISABLE_TAGS = ConfigEntry(LegacyConfigEntry(SECTION, "disable_tags", bool))
def test_config_entry_file():
# Pytest feature
c = ConfigEntry(LegacyConfigEntry("platform", "url", str))
assert c.read() is None
cfg = get_config_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs/good.config"))
assert c.read(cfg) == "fakeflyte.com"
c = ConfigEntry(LegacyConfigEntry("platform", "url2", str)) # Does not exist
assert c.read(cfg) is None
def test_config_entry_file():
c = ConfigEntry(LegacyConfigEntry("platform", "url", str),
YamlConfigEntry("admin.endpoint"),
lambda x: x.replace("dns:///", ""))
assert c.read() is None
cfg = get_config_file(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"configs/sample.yaml"))
assert c.read(cfg) == "flyte.mycorp.io"
c = ConfigEntry(LegacyConfigEntry("platform", "url2",
str)) # Does not exist
assert c.read(cfg) is None
class LocalSDK(object):
SECTION = "sdk"
WORKFLOW_PACKAGES = ConfigEntry(LegacyConfigEntry(SECTION, "workflow_packages", list))
"""
This is a comma-delimited list of packages that SDK tools will use to discover entities for the purpose of registration
and execution of entities.
"""
LOCAL_SANDBOX = ConfigEntry(LegacyConfigEntry(SECTION, "local_sandbox"))
"""
This is the path where SDK will place files during local executions and testing. The SDK will not automatically
clean up data in these directories.
"""
LOGGING_LEVEL = ConfigEntry(LegacyConfigEntry(SECTION, "logging_level", int))
"""
def test_env_var_bool_transformer(mock_file_read):
mock_file_read.return_value = None
test_env_var = "FLYTE_MADEUP_TEST_VAR_ABC123"
b = ConfigEntry(LegacyConfigEntry("madeup", "test_var_abc123", bool))
os.environ[test_env_var] = "FALSE"
assert b.read() is False
os.environ[test_env_var] = ""
assert b.read() is False
os.environ[test_env_var] = "1"
assert b.read()
os.environ[test_env_var] = "truee"
assert b.read()
# The above reads shouldn't have triggered the file read since the env var was set
assert mock_file_read.call_count == 0
del os.environ[test_env_var]
cfg_file_mock = mock.MagicMock()
cfg_file_mock.legacy_config.return_value = True
assert b.read(cfg=cfg_file_mock) is None
# The last read should've triggered the file read since now the env var is no longer set.
assert mock_file_read.call_count == 1
class Secrets(object):
SECTION = "secrets"
# Secrets management
ENV_PREFIX = ConfigEntry(LegacyConfigEntry(SECTION, "env_prefix"))
"""
This is the prefix that will be used to lookup for injected secrets at runtime. This can be overridden to using
FLYTE_SECRETS_ENV_PREFIX variable
"""
DEFAULT_DIR = ConfigEntry(LegacyConfigEntry(SECTION, "default_dir"))
"""
This is the default directory that will be used to find secrets as individual files under. This can be overridden using
FLYTE_SECRETS_DEFAULT_DIR.
"""
FILE_PREFIX = ConfigEntry(LegacyConfigEntry(SECTION, "file_prefix"))
"""
def test_config_entry_envvar():
# Pytest feature
c = ConfigEntry(LegacyConfigEntry("test", "op1", str))
assert c.read() is None
old_environ = dict(os.environ)
os.environ["FLYTE_TEST_OP1"] = "xyz"
assert c.read() == "xyz"
os.environ = old_environ
def test_config_entry_file_2(mock_get):
# Test reading of the environment variable that flytectl asks users to set.
# Can take both extensions
sample_yaml_file_name = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "configs/sample.yml")
mock_get.return_value = sample_yaml_file_name
c = ConfigEntry(LegacyConfigEntry("platform", "url", str),
YamlConfigEntry("admin.endpoint"),
lambda x: x.replace("dns:///", ""))
assert c.read() is None
cfg = get_config_file(sample_yaml_file_name)
assert c.read(cfg) == "flyte.mycorp.io"
c = ConfigEntry(LegacyConfigEntry("platform", "url2",
str)) # Does not exist
assert c.read(cfg) is None
def test_config_entry_types():
cfg = get_config_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs/good.config"))
l = ConfigEntry(LegacyConfigEntry("sdk", "workflow_packages", list))
assert l.read(cfg) == ["this.module", "that.module"]
s = ConfigEntry(LegacyConfigEntry("madeup", "string_value"))
assert s.read(cfg) == "abc"
i = ConfigEntry(LegacyConfigEntry("madeup", "int_value", int))
assert i.read(cfg) == 3
b = ConfigEntry(LegacyConfigEntry("madeup", "bool_value", bool))
assert b.read(cfg) is False
t = ConfigEntry(
LegacyConfigEntry("madeup", "timedelta_value", datetime.timedelta),
transform=lambda x: datetime.timedelta(seconds=timeparse(x)),
)
assert t.read(cfg) == datetime.timedelta(hours=20)
def test_config_entry_precedence():
# Pytest feature
c = ConfigEntry(LegacyConfigEntry("platform", "url", str))
assert c.read() is None
old_environ = dict(os.environ)
os.environ["FLYTE_PLATFORM_URL"] = "xyz"
cfg = get_config_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs/good.config"))
assert c.read(cfg) == "xyz"
# reset
os.environ = old_environ
class Credentials(object):
SECTION = "credentials"
COMMAND = ConfigEntry(LegacyConfigEntry(SECTION, "command", list), YamlConfigEntry("admin.command", list))
"""
This command is executed to return a token using an external process.
"""
CLIENT_ID = ConfigEntry(LegacyConfigEntry(SECTION, "client_id"), YamlConfigEntry("admin.clientId"))
"""
This is the public identifier for the app which handles authorization for a Flyte deployment.
More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
"""
CLIENT_CREDENTIALS_SECRET = ConfigEntry(LegacyConfigEntry(SECTION, "client_secret"))
"""
Used for basic auth, which is automatically called during pyflyte. This will allow the Flyte engine to read the
password directly from the environment variable. Note that this is less secure! Please only use this if mounting the
secret as a file is impossible.
"""
CLIENT_CREDENTIALS_SECRET_LOCATION = ConfigEntry(
LegacyConfigEntry(SECTION, "client_secret_location"), YamlConfigEntry("admin.clientSecretLocation")
)
"""
Used for basic auth, which is automatically called during pyflyte. This will allow the Flyte engine to read the
password from a mounted file.
"""
SCOPES = ConfigEntry(LegacyConfigEntry(SECTION, "scopes", list), YamlConfigEntry("admin.scopes", list))
AUTH_MODE = ConfigEntry(LegacyConfigEntry(SECTION, "auth_mode"), YamlConfigEntry("admin.authType"))
"""
class AWS(object):
SECTION = "aws"
S3_ENDPOINT = ConfigEntry(LegacyConfigEntry(SECTION, "endpoint"), YamlConfigEntry("storage.connection.endpoint"))
S3_ACCESS_KEY_ID = ConfigEntry(
LegacyConfigEntry(SECTION, "access_key_id"), YamlConfigEntry("storage.connection.access-key")
)
S3_SECRET_ACCESS_KEY = ConfigEntry(
LegacyConfigEntry(SECTION, "secret_access_key"), YamlConfigEntry("storage.connection.secret-key")
)
ENABLE_DEBUG = ConfigEntry(LegacyConfigEntry(SECTION, "enable_debug", bool))
RETRIES = ConfigEntry(LegacyConfigEntry(SECTION, "retries", int))
BACKOFF_SECONDS = ConfigEntry(
LegacyConfigEntry(SECTION, "backoff_seconds", datetime.timedelta),
transform=lambda x: datetime.timedelta(seconds=int(x)),
)
class GCP(object):
SECTION = "gcp"
GSUTIL_PARALLELISM = ConfigEntry(LegacyConfigEntry(SECTION, "gsutil_parallelism", bool))
class Deck(object):
SECTION = "deck"
DISABLE_DECK = ConfigEntry(LegacyConfigEntry(SECTION, "disable_deck", bool))