def get_specified_images() -> typing.Dict[str, str]:
"""
This section should contain options, where the option name is the friendly name of the image and the corresponding
value is actual FQN of the image. Example of how the section is structured
[images]
my_image1=docker.io/flyte:tag
# Note that the tag is optional. If not specified it will be the default version identifier specified
my_image2=docker.io/flyte
:returns a dictionary of name: image<fqn+version> Version is optional
"""
images: typing.Dict[str, str] = {}
if _config_common.CONFIGURATION_SINGLETON.config is None:
return images
try:
image_names = _config_common.CONFIGURATION_SINGLETON.config.options(
"images")
except configparser.NoSectionError:
print("No images specified, will use the default image")
image_names = None
if image_names:
for i in image_names:
images[str(i)] = _config_common.FlyteStringConfigurationEntry(
"images", i).get()
return images
def test_lookup_waterfall_raw_env_var():
x = _common.FlyteStringConfigurationEntry('test', 'setting', default=None)
if 'FLYTE_TEST_SETTING' in _os.environ:
del _os.environ['FLYTE_TEST_SETTING']
assert x.get() is None
_os.environ['FLYTE_TEST_SETTING'] = 'lorem'
assert x.get() == 'lorem'
def test_lookup_waterfall_raw_env_var():
x = _common.FlyteStringConfigurationEntry("test", "setting", default=None)
if "FLYTE_TEST_SETTING" in _os.environ:
del _os.environ["FLYTE_TEST_SETTING"]
assert x.get() is None
_os.environ["FLYTE_TEST_SETTING"] = "lorem"
assert x.get() == "lorem"
def test_env_var_precedence_string():
set_flyte_config_file(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'configs/good.config'))
assert common.FlyteIntegerConfigurationEntry('madeup',
'int_value').get() == 3
assert common.FlyteStringConfigurationEntry('madeup',
'string_value').get() == 'abc'
old_environ = dict(os.environ)
try:
os.environ['FLYTE_MADEUP_INT_VALUE'] = '10'
os.environ["FLYTE_MADEUP_STRING_VALUE"] = 'overridden'
assert common.FlyteIntegerConfigurationEntry('madeup',
'int_value').get() == 10
assert common.FlyteStringConfigurationEntry(
'madeup', 'string_value').get() == 'overridden'
finally:
os.environ.clear()
os.environ.update(old_environ)
def test_lookup_waterfall_referenced_env_var():
x = _common.FlyteStringConfigurationEntry('test', 'setting', default=None)
if 'FLYTE_TEST_SETTING' in _os.environ:
del _os.environ['FLYTE_TEST_SETTING']
assert x.get() is None
if 'TEMP_PLACEHOLDER' in _os.environ:
del _os.environ['TEMP_PLACEHOLDER']
_os.environ['TEMP_PLACEHOLDER'] = 'lorem'
_os.environ['FLYTE_TEST_SETTING_FROM_ENV_VAR'] = 'TEMP_PLACEHOLDER'
assert x.get() == 'lorem'
def test_env_var_precedence_string():
set_flyte_config_file(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"configs/good.config"))
assert common.FlyteIntegerConfigurationEntry("madeup",
"int_value").get() == 3
assert common.FlyteStringConfigurationEntry("madeup",
"string_value").get() == "abc"
old_environ = dict(os.environ)
try:
os.environ["FLYTE_MADEUP_INT_VALUE"] = "10"
os.environ["FLYTE_MADEUP_STRING_VALUE"] = "overridden"
assert common.FlyteIntegerConfigurationEntry("madeup",
"int_value").get() == 10
assert common.FlyteStringConfigurationEntry(
"madeup", "string_value").get() == "overridden"
finally:
os.environ.clear()
os.environ.update(old_environ)
def test_lookup_waterfall_referenced_env_var():
x = _common.FlyteStringConfigurationEntry("test", "setting", default=None)
if "FLYTE_TEST_SETTING" in _os.environ:
del _os.environ["FLYTE_TEST_SETTING"]
assert x.get() is None
if "TEMP_PLACEHOLDER" in _os.environ:
del _os.environ["TEMP_PLACEHOLDER"]
_os.environ["TEMP_PLACEHOLDER"] = "lorem"
_os.environ["FLYTE_TEST_SETTING_FROM_ENV_VAR"] = "TEMP_PLACEHOLDER"
assert x.get() == "lorem"
def test_lookup_waterfall_referenced_file():
x = _common.FlyteStringConfigurationEntry("test", "setting", default=None)
if "FLYTE_TEST_SETTING" in _os.environ:
del _os.environ["FLYTE_TEST_SETTING"]
assert x.get() is None
with _AutoDeletingTempDir("config_testing") as tmp_dir:
with open(tmp_dir.get_named_tempfile("name"), "w") as fh:
fh.write("secret_password")
_os.environ[
"FLYTE_TEST_SETTING_FROM_FILE"] = tmp_dir.get_named_tempfile(
"name")
assert x.get() == "secret_password"
def test_lookup_waterfall_referenced_file():
x = _common.FlyteStringConfigurationEntry('test', 'setting', default=None)
if 'FLYTE_TEST_SETTING' in _os.environ:
del _os.environ['FLYTE_TEST_SETTING']
assert x.get() is None
with _AutoDeletingTempDir("config_testing") as tmp_dir:
with open(tmp_dir.get_named_tempfile('name'), 'w') as fh:
fh.write('secret_password')
_os.environ[
'FLYTE_TEST_SETTING_FROM_FILE'] = tmp_dir.get_named_tempfile(
'name')
assert x.get() == 'secret_password'
from __future__ import absolute_import
from flytekit.configuration import common as _common_config
HOST = _common_config.FlyteStringConfigurationEntry('statsd', 'host', default='localhost')
PORT = _common_config.FlyteIntegerConfigurationEntry('statsd', 'port', default=8125)
from __future__ import absolute_import
from flytekit.configuration import common as _config_common
WORKFLOW_PACKAGES = _config_common.FlyteStringListConfigurationEntry('sdk', 'workflow_packages', default=[])
"""
This is a comma-delimited list of packages that SDK tools will use to discover entities for the purpose of registration
and execution of entities.
"""
EXECUTION_ENGINE = _config_common.FlyteStringConfigurationEntry('sdk', 'execution_engine', default='flyte')
"""
This is a comma-delimited list of package strings, in order, for resolving execution behavior.
TODO: Explain how this would be used to extend the SDK
"""
TYPE_ENGINES = _config_common.FlyteStringListConfigurationEntry('sdk', 'type_engines', default=[])
"""
This is a comma-delimited list of package strings, in order, for resolving type behavior.
TODO: Explain how this would be used to extend the SDK
"""
LOCAL_SANDBOX = _config_common.FlyteStringConfigurationEntry('sdk', 'local_sandbox', default="/tmp/flyte")
"""
This is the path where SDK will place files during local executions and testing. The SDK will not automatically
clean up data in these directories.
"""
SDK_PYTHON_VENV = _config_common.FlyteStringListConfigurationEntry('sdk', 'python_venv', default=[])
from flytekit.configuration import common as _config_common
ASSUMABLE_IAM_ROLE = _config_common.FlyteStringConfigurationEntry(
"auth", "assumable_iam_role", default=None)
"""
This is the role the SDK will use by default to execute workflows. For example, in AWS this should be an IAM role
string.
"""
KUBERNETES_SERVICE_ACCOUNT = _config_common.FlyteStringConfigurationEntry(
"auth", "kubernetes_service_account", default=None)
"""
This is the kubernetes service account that will be passed to workflow executions.
"""
RAW_OUTPUT_DATA_PREFIX = _config_common.FlyteStringConfigurationEntry(
"auth", "raw_output_data_prefix", default="")
"""
This is not output metadata but rather where users can specify an S3 or gcs path for offloaded data like blobs
and schemas.
The reason this setting is in this file is because it's inextricably tied to a workflow's role or service account,
since that is what ultimately gives the tasks the ability to write to certain buckets.
"""
from flytekit.configuration import common as _config_common
S3_SHARD_FORMATTER = _config_common.FlyteRequiredStringConfigurationEntry(
"aws", "s3_shard_formatter")
S3_SHARD_STRING_LENGTH = _config_common.FlyteIntegerConfigurationEntry(
"aws", "s3_shard_string_length", default=2)
S3_ENDPOINT = _config_common.FlyteStringConfigurationEntry("aws",
"endpoint",
default=None)
S3_ACCESS_KEY_ID = _config_common.FlyteStringConfigurationEntry(
"aws", "access_key_id", default=None)
S3_SECRET_ACCESS_KEY = _config_common.FlyteStringConfigurationEntry(
"aws", "secret_access_key", default=None)
S3_ACCESS_KEY_ID_ENV_NAME = "AWS_ACCESS_KEY_ID"
S3_SECRET_ACCESS_KEY_ENV_NAME = "AWS_SECRET_ACCESS_KEY"
S3_ENDPOINT_ARG_NAME = "--endpoint-url"
ENABLE_DEBUG = _config_common.FlyteBoolConfigurationEntry("aws",
"enable_debug",
default=False)
RETRIES = _config_common.FlyteIntegerConfigurationEntry("aws",
"retries",
default=3)
from flytekit.configuration import common as _common_config
HOST = _common_config.FlyteStringConfigurationEntry("statsd",
"host",
default="localhost")
PORT = _common_config.FlyteIntegerConfigurationEntry("statsd",
"port",
default=8125)
DISABLED = _common_config.FlyteBoolConfigurationEntry("statsd",
"disabled",
default=False)
import os
from flytekit.configuration import common as _common_config
# Secrets management
SECRETS_ENV_PREFIX = _common_config.FlyteStringConfigurationEntry(
"secrets", "env_prefix", default="_FSEC_")
"""
This is the prefix that will be used to lookup for injected secrets at runtime. This can be overriden to using
FLYTE_SECRETS_ENV_PREFIX variable
"""
SECRETS_DEFAULT_DIR = _common_config.FlyteStringConfigurationEntry(
"secrets", "default_dir", default=os.path.join(os.sep, "etc", "secrets"))
"""
This is the default directory that will be used to find secrets as individual files under. This can be overriden using
FLYTE_SECRETS_DEFAULT_DIR.
"""
SECRETS_FILE_PREFIX = _common_config.FlyteStringConfigurationEntry(
"secrets", "file_prefix", default="")
"""
This is the prefix for the file in the default dir.
"""
from __future__ import absolute_import
from flytekit.configuration import common as _config_common
S3_SHARD_FORMATTER = _config_common.FlyteRequiredStringConfigurationEntry(
'aws', 's3_shard_formatter')
S3_SHARD_STRING_LENGTH = _config_common.FlyteIntegerConfigurationEntry(
'aws', 's3_shard_string_length', default=2)
S3_ENDPOINT = _config_common.FlyteStringConfigurationEntry('aws',
'endpoint',
default=None)
S3_ACCESS_KEY_ID = _config_common.FlyteStringConfigurationEntry(
'aws', 'access_key_id', default=None)
S3_SECRET_ACCESS_KEY = _config_common.FlyteStringConfigurationEntry(
'aws', 'secret_access_key', default=None)
S3_ACCESS_KEY_ID_ENV_NAME = 'AWS_ACCESS_KEY_ID'
S3_SECRET_ACCESS_KEY_ENV_NAME = 'AWS_SECRET_ACCESS_KEY'
S3_ENDPOINT_ARG_NAME = '--endpoint-url'
from flytekit.configuration import common as _config_common
WORKFLOW_PACKAGES = _config_common.FlyteStringListConfigurationEntry("sdk", "workflow_packages", default=[])
"""
This is a comma-delimited list of packages that SDK tools will use to discover entities for the purpose of registration
and execution of entities.
"""
EXECUTION_ENGINE = _config_common.FlyteStringConfigurationEntry("sdk", "execution_engine", default="flyte")
"""
This is a comma-delimited list of package strings, in order, for resolving execution behavior.
TODO: Explain how this would be used to extend the SDK
"""
TYPE_ENGINES = _config_common.FlyteStringListConfigurationEntry("sdk", "type_engines", default=[])
"""
This is a comma-delimited list of package strings, in order, for resolving type behavior.
TODO: Explain how this would be used to extend the SDK
"""
LOCAL_SANDBOX = _config_common.FlyteStringConfigurationEntry("sdk", "local_sandbox", default="/tmp/flyte")
"""
This is the path where SDK will place files during local executions and testing. The SDK will not automatically
clean up data in these directories.
"""
SDK_PYTHON_VENV = _config_common.FlyteStringListConfigurationEntry("sdk", "python_venv", default=[])
"""
This is a list of commands/args which will be prefixed to the entrypoint command by SDK.
import re
from flytekit.configuration import common as _common_config
IMAGE = _common_config.FlyteStringConfigurationEntry("internal", "image")
# This configuration option specifies the path to the file that holds the configuration options. Don't worry,
# there will not be cycles because the parsing of the configuration file intentionally will not read and settings
# in the [internal] section.
# The default, if you want to use it, should be a file called flytekit.config, located in wherever your python
# interpreter originates.
CONFIGURATION_PATH = _common_config.FlyteStringConfigurationEntry(
"internal", "configuration_path", default="flytekit.config")
# Project, Domain and Version represent the values at registration time.
PROJECT = _common_config.FlyteStringConfigurationEntry("internal",
"project",
default="")
DOMAIN = _common_config.FlyteStringConfigurationEntry("internal",
"domain",
default="")
NAME = _common_config.FlyteStringConfigurationEntry("internal",
"name",
default="")
VERSION = _common_config.FlyteStringConfigurationEntry("internal",
"version",
default="")
# Project, Domain and Version represent the values at registration time.
TASK_PROJECT = _common_config.FlyteStringConfigurationEntry("internal",
"task_project",
default="")
from __future__ import absolute_import
from flytekit.configuration import common as _config_common
CLIENT_ID = _config_common.FlyteStringConfigurationEntry('credentials',
'client_id',
default=None)
"""
This is the public identifier for the app which handles authorization for a Flyte deployment.
More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
"""
REDIRECT_URI = _config_common.FlyteStringConfigurationEntry(
'credentials', 'redirect_uri', default="http://localhost:12345/callback")
"""
This is the callback uri registered with the app which handles authorization for a Flyte deployment.
Please note the hardcoded port number. Ideally we would not do this, but some IDPs do not allow wildcards for
the URL, which means we have to use the same port every time. This is the only reason this is a configuration option,
otherwise, we'd just hardcode the callback path as a constant.
FYI, to see if a given port is already in use, run `sudo lsof -i :<port>` if on a Linux system.
More details here: https://www.oauth.com/oauth2-servers/redirect-uris/.
"""
AUTHORIZATION_METADATA_KEY = _config_common.FlyteStringConfigurationEntry(
'credentials', 'authorization_metadata_key', default="authorization")
"""
The authorization metadata key used for passing access tokens in gRPC requests.
Traditionally this value is 'authorization' however it is made configurable.
"""
CLIENT_CREDENTIALS_SECRET = _config_common.FlyteStringConfigurationEntry(
from warnings import warn
from flytekit.configuration import common as _config_common
deprecated_names = ["CLIENT_CREDENTIALS_SCOPE"]
CLIENT_ID = _config_common.FlyteStringConfigurationEntry("credentials",
"client_id",
default=None)
"""
This is the public identifier for the app which handles authorization for a Flyte deployment.
More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
"""
REDIRECT_URI = _config_common.FlyteStringConfigurationEntry(
"credentials", "redirect_uri", default="http://localhost:12345/callback")
"""
This is the callback uri registered with the app which handles authorization for a Flyte deployment.
Please note the hardcoded port number. Ideally we would not do this, but some IDPs do not allow wildcards for
the URL, which means we have to use the same port every time. This is the only reason this is a configuration option,
otherwise, we'd just hardcode the callback path as a constant.
FYI, to see if a given port is already in use, run `sudo lsof -i :<port>` if on a Linux system.
More details here: https://www.oauth.com/oauth2-servers/redirect-uris/.
"""
OAUTH_SCOPES = _config_common.FlyteStringListConfigurationEntry(
"credentials", "oauth_scopes", default=["openid"])
"""
This controls the list of scopes to request from the authorization server.
"""
from __future__ import absolute_import
from flytekit.configuration import common as _config_common
from flytekit.common import constants as _constants
URL = _config_common.FlyteRequiredStringConfigurationEntry('platform', 'url')
INSECURE = _config_common.FlyteBoolConfigurationEntry('platform',
'insecure',
default=False)
CLOUD_PROVIDER = _config_common.FlyteStringConfigurationEntry(
'platform', 'cloud_provider', default=_constants.CloudProvider.AWS)
from flytekit.common import constants as _constants
from flytekit.configuration import common as _config_common
URL = _config_common.FlyteStringConfigurationEntry("platform", "url")
HTTP_URL = _config_common.FlyteStringConfigurationEntry("platform", "http_url", default=None)
"""
If not starting with either http or https, this setting should begin with // as per the urlparse library and
https://tools.ietf.org/html/rfc1808.html, otherwise the netloc will not be properly parsed.
Currently the only use-case for this configuration setting is for Auth discovery. This setting supports the case where
Flyte Admin's gRPC and HTTP points are deployed on different ports.
"""
INSECURE = _config_common.FlyteBoolConfigurationEntry("platform", "insecure", default=False)
CLOUD_PROVIDER = _config_common.FlyteStringConfigurationEntry(
"platform", "cloud_provider", default=_constants.CloudProvider.AWS
)
AUTH = _config_common.FlyteBoolConfigurationEntry("platform", "auth", default=False)
"""
This config setting should not normally be filled in. Whether or not an admin server requires authentication should be
something published by the admin server itself (typically by returning a 401). However, to help with migration, this
config object is here to force the SDK to attempt the auth flow even without prompting by Admin.
"""
from flytekit.configuration import common as _config_common
DEFAULT_CPU_LIMIT = _config_common.FlyteStringConfigurationEntry(
"resources", "default_cpu_limit")
"""
If not specified explicitly when constructing a task, this limit will be applied as the default. Follows Kubernetes CPU
request/limit format.
"""
DEFAULT_CPU_REQUEST = _config_common.FlyteStringConfigurationEntry(
"resources", "default_cpu_request")
"""
If not specified explicitly when constructing a task, this request will be applied as the default. Follows Kubernetes
CPU request/limit format.
"""
DEFAULT_MEMORY_LIMIT = _config_common.FlyteStringConfigurationEntry(
"resources", "default_memory_limit")
"""
If not specified explicitly when constructing a task, this limit will be applied as the default. Follows Kubernetes
memory request/limit format.
"""
DEFAULT_MEMORY_REQUEST = _config_common.FlyteStringConfigurationEntry(
"resources", "default_memory_request")
"""
If not specified explicitly when constructing a task, this request will be applied as the default. Follows Kubernetes
memory request/limit format.
"""
DEFAULT_GPU_LIMIT = _config_common.FlyteStringConfigurationEntry(
from __future__ import absolute_import
import re
from flytekit.configuration import common as _common_config
IMAGE = _common_config.FlyteRequiredStringConfigurationEntry(
'internal', 'image')
# This configuration option specifies the path to the file that holds the configuration options. Don't worry,
# there will not be cycles because the parsing of the configuration file intentionally will not read and settings
# in the [internal] section.
# The default, if you want to use it, should be a file called flytekit.config, located in wherever your python
# interpreter originates.
CONFIGURATION_PATH = _common_config.FlyteStringConfigurationEntry(
'internal', 'configuration_path', default='flytekit.config')
# Project, Domain and Version represent the values at registration time.
PROJECT = _common_config.FlyteStringConfigurationEntry('internal',
'project',
default="")
DOMAIN = _common_config.FlyteStringConfigurationEntry('internal',
'domain',
default="")
NAME = _common_config.FlyteStringConfigurationEntry('internal',
'name',
default="")
VERSION = _common_config.FlyteStringConfigurationEntry('internal',
'version',
default="")
# Project, Domain and Version represent the values at registration time.
